@message-queue-toolkit/kafka 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AbstractKafkaConsumer.d.ts +2 -2
- package/dist/AbstractKafkaConsumer.js +6 -2
- package/dist/AbstractKafkaConsumer.js.map +1 -1
- package/dist/utils/safeJsonDeserializer.d.ts +1 -0
- package/dist/utils/safeJsonDeserializer.js +14 -0
- package/dist/utils/safeJsonDeserializer.js.map +1 -0
- package/package.json +1 -1
|
@@ -2,9 +2,9 @@ import type { QueueConsumerDependencies } from '@message-queue-toolkit/core';
|
|
|
2
2
|
import { type ConsumeOptions, type ConsumerOptions } from '@platformatic/kafka';
|
|
3
3
|
import { AbstractKafkaService, type BaseKafkaOptions } from './AbstractKafkaService.ts';
|
|
4
4
|
import type { KafkaHandlerRouting } from './handler-container/KafkaHandlerRoutingBuilder.ts';
|
|
5
|
-
import type { KafkaDependencies, TopicConfig } from './types.ts';
|
|
5
|
+
import type { KafkaConfig, KafkaDependencies, TopicConfig } from './types.ts';
|
|
6
6
|
export type KafkaConsumerDependencies = KafkaDependencies & Pick<QueueConsumerDependencies, 'transactionObservabilityManager'>;
|
|
7
|
-
export type KafkaConsumerOptions<TopicsConfig extends TopicConfig[]> = BaseKafkaOptions & Omit<ConsumerOptions<string, object, string, string>, 'deserializers' | 'autocommit' |
|
|
7
|
+
export type KafkaConsumerOptions<TopicsConfig extends TopicConfig[]> = BaseKafkaOptions & Omit<ConsumerOptions<string, object, string, string>, 'deserializers' | 'autocommit' | keyof KafkaConfig> & Omit<ConsumeOptions<string, object, string, string>, 'topics'> & {
|
|
8
8
|
handlers: KafkaHandlerRouting<TopicsConfig>;
|
|
9
9
|
};
|
|
10
10
|
export declare abstract class AbstractKafkaConsumer<TopicsConfig extends TopicConfig[]> extends AbstractKafkaService<TopicsConfig, KafkaConsumerOptions<TopicsConfig>> {
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { setTimeout } from 'node:timers/promises';
|
|
3
3
|
import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
|
|
4
|
-
import { Consumer,
|
|
4
|
+
import { Consumer, stringDeserializer, } from '@platformatic/kafka';
|
|
5
5
|
import { AbstractKafkaService } from "./AbstractKafkaService.js";
|
|
6
6
|
import { KafkaHandlerContainer } from "./handler-container/KafkaHandlerContainer.js";
|
|
7
|
+
import { safeJsonDeserializer } from './utils/safeJsonDeserializer.js';
|
|
7
8
|
/*
|
|
8
9
|
TODO: Proper retry mechanism + DLQ -> https://lokalise.atlassian.net/browse/EDEXP-498
|
|
9
10
|
In the meantime, we will retry in memory up to 3 times
|
|
@@ -24,7 +25,7 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
24
25
|
autocommit: false, // Handling commits manually
|
|
25
26
|
deserializers: {
|
|
26
27
|
key: stringDeserializer,
|
|
27
|
-
value:
|
|
28
|
+
value: safeJsonDeserializer,
|
|
28
29
|
headerKey: stringDeserializer,
|
|
29
30
|
headerValue: stringDeserializer,
|
|
30
31
|
},
|
|
@@ -58,6 +59,9 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
58
59
|
await this.consumer.close();
|
|
59
60
|
}
|
|
60
61
|
async consume(message) {
|
|
62
|
+
// message.value can be undefined if the message is not JSON-serializable
|
|
63
|
+
if (!message.value)
|
|
64
|
+
return message.commit();
|
|
61
65
|
const handler = this.handlerContainer.resolveHandler(message.topic, message.value);
|
|
62
66
|
// if there is no handler for the message, we ignore it (simulating subscription)
|
|
63
67
|
if (!handler)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EAEb,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAEL,QAAQ,EAIR,
|
|
1
|
+
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EAEb,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAEL,QAAQ,EAIR,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAyB,MAAM,2BAA2B,CAAA;AACvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAA;AAIpF,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AActE;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAEpB,SAAQ,oBAAsE;IAC7D,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAEtD,+BAA+B,CAAiC;IAChE,gBAAgB,CAAqC;IAEtE,YACE,YAAuC,EACvC,OAA2C;QAE3C,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,IAAI,qBAAqB,CAC/C,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,gBAAgB,CACzB,CAAA;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAA;QAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YACnG,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;QAClF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;QAClE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QAElD,MAAM,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAgD;QACpE,yEAAyE;QACzE,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QAE3C,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;QAClF,iFAAiF;QACjF,IAAI,CAAC,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QAErC,oBAAoB;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,UAAU,EAAE,CAAA;QAC1E,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC,CAAA;QAE9F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC3D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;gBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;YACF,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,OAAO,CAAC,KAAK;gBACtB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;aACrE,CAAC,CAAA;YAEF,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,gBAAgB,GAAG,WAAW,CAAC,IAAI,CAAA;QAEzC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;QAEtD,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,QAAQ,GAAG,KAAK,CAAA;QACpB,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAChC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,EACvC,OAAO,CAAC,OAAO,EACf,cAAc,CACf,CAAA;YACD,IAAI,QAAQ;gBAAE,MAAK;YAEnB,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;aACzC,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE;aACnE,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,OAAsD,EACtD,OAAmC,EACnC,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;YACtC,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,oBAAoB,CAAC,OAAgD;QAC3E,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAE1D,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,KAAK,EAAE,CAAA;QACnC,IAAI,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM;YAAE,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE,CAAA;QAEhE,OAAO,IAAI,CAAA;IACb,CAAC;IAEO,iBAAiB,CAAC,OAAgD;QACxE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const safeJsonDeserializer: (data?: string | Buffer) => object | undefined;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export const safeJsonDeserializer = (data) => {
|
|
2
|
+
if (!data)
|
|
3
|
+
return undefined;
|
|
4
|
+
// Checking to be safe
|
|
5
|
+
if (!Buffer.isBuffer(data) && typeof data !== 'string')
|
|
6
|
+
return undefined;
|
|
7
|
+
try {
|
|
8
|
+
return JSON.parse(data.toString('utf-8'));
|
|
9
|
+
}
|
|
10
|
+
catch (_) {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
//# sourceMappingURL=safeJsonDeserializer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safeJsonDeserializer.js","sourceRoot":"","sources":["../../lib/utils/safeJsonDeserializer.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAsB,EAAsB,EAAE;IACjF,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAC3B,sBAAsB;IACtB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAExE,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAC3C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|