@message-queue-toolkit/kafka 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { setTimeout } from 'node:timers/promises';
|
|
3
3
|
import { InternalError, stringValueSerializer, } from '@lokalise/node-core';
|
|
4
|
-
import { Consumer,
|
|
4
|
+
import { Consumer, stringDeserializer, } from '@platformatic/kafka';
|
|
5
5
|
import { AbstractKafkaService } from "./AbstractKafkaService.js";
|
|
6
6
|
import { KafkaHandlerContainer } from "./handler-container/KafkaHandlerContainer.js";
|
|
7
|
+
import { safeJsonDeserializer } from './utils/safeJsonDeserializer.js';
|
|
7
8
|
/*
|
|
8
9
|
TODO: Proper retry mechanism + DLQ -> https://lokalise.atlassian.net/browse/EDEXP-498
|
|
9
10
|
In the meantime, we will retry in memory up to 3 times
|
|
@@ -24,7 +25,7 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
24
25
|
autocommit: false, // Handling commits manually
|
|
25
26
|
deserializers: {
|
|
26
27
|
key: stringDeserializer,
|
|
27
|
-
value:
|
|
28
|
+
value: safeJsonDeserializer,
|
|
28
29
|
headerKey: stringDeserializer,
|
|
29
30
|
headerValue: stringDeserializer,
|
|
30
31
|
},
|
|
@@ -58,6 +59,9 @@ export class AbstractKafkaConsumer extends AbstractKafkaService {
|
|
|
58
59
|
await this.consumer.close();
|
|
59
60
|
}
|
|
60
61
|
async consume(message) {
|
|
62
|
+
// message.value can be undefined if the message is not JSON-serializable
|
|
63
|
+
if (!message.value)
|
|
64
|
+
return message.commit();
|
|
61
65
|
const handler = this.handlerContainer.resolveHandler(message.topic, message.value);
|
|
62
66
|
// if there is no handler for the message, we ignore it (simulating subscription)
|
|
63
67
|
if (!handler)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EAEb,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAEL,QAAQ,EAIR,
|
|
1
|
+
{"version":3,"file":"AbstractKafkaConsumer.js","sourceRoot":"","sources":["../lib/AbstractKafkaConsumer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EACL,aAAa,EAEb,qBAAqB,GACtB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EAEL,QAAQ,EAIR,kBAAkB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,oBAAoB,EAAyB,MAAM,2BAA2B,CAAA;AACvF,OAAO,EAAE,qBAAqB,EAAE,MAAM,8CAA8C,CAAA;AAIpF,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAA;AActE;;;GAGG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B,MAAM,OAAgB,qBAEpB,SAAQ,oBAAsE;IAC7D,QAAQ,CAA0C;IAC3D,cAAc,CAAiD;IAEtD,+BAA+B,CAAiC;IAChE,gBAAgB,CAAqC;IAEtE,YACE,YAAuC,EACvC,OAA2C;QAE3C,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAA;QAE5B,IAAI,CAAC,+BAA+B,GAAG,YAAY,CAAC,+BAA+B,CAAA;QACnF,IAAI,CAAC,gBAAgB,GAAG,IAAI,qBAAqB,CAC/C,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,gBAAgB,CACzB,CAAA;QAED,IAAI,CAAC,QAAQ,GAAG,IAAI,QAAQ,CAAC;YAC3B,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK;YACrB,GAAG,IAAI,CAAC,OAAO;YACf,UAAU,EAAE,KAAK,EAAE,4BAA4B;YAC/C,aAAa,EAAE;gBACb,GAAG,EAAE,kBAAkB;gBACvB,KAAK,EAAE,oBAAoB;gBAC3B,SAAS,EAAE,kBAAkB;gBAC7B,WAAW,EAAE,kBAAkB;aAChC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAA;QAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAE9E,IAAI,CAAC;YACH,MAAM,EAAE,QAAQ,EAAE,GAAG,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA,CAAC,8CAA8C;YACnG,IAAI,CAAC,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,GAAG,cAAc,EAAE,MAAM,EAAE,CAAC,CAAA;QAClF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,aAAa,CAAC;gBACtB,OAAO,EAAE,sBAAsB;gBAC/B,SAAS,EAAE,2BAA2B;gBACtC,KAAK,EAAE,KAAK;aACb,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAA;QAClE,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAA;IACtE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAA;QAElD,MAAM,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC7D,IAAI,CAAC,cAAc,GAAG,SAAS,CAAA;QAC/B,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAC7B,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,OAAgD;QACpE,yEAAyE;QACzE,IAAI,CAAC,OAAO,CAAC,KAAK;YAAE,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QAE3C,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;QAClF,iFAAiF;QACjF,IAAI,CAAC,OAAO;YAAE,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QAErC,oBAAoB;QACpB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,UAAU,EAAE,CAAA;QAC1E,IAAI,CAAC,+BAA+B,EAAE,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC,CAAA;QAE9F,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAC3D,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,KAAK,EAAE;gBACnC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;YACF,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,OAAO,CAAC,KAAK;gBACtB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE;aACrE,CAAC,CAAA;YAEF,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;QACzB,CAAC;QAED,MAAM,gBAAgB,GAAG,WAAW,CAAC,IAAI,CAAA;QAEzC,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAA;QAEtD,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,QAAQ,GAAG,KAAK,CAAA;QACpB,GAAG,CAAC;YACF,qCAAqC;YACrC,IAAI,OAAO,GAAG,CAAC;gBAAE,MAAM,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAA;YAE3D,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAChC,EAAE,GAAG,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAE,EACvC,OAAO,CAAC,OAAO,EACf,cAAc,CACf,CAAA;YACD,IAAI,QAAQ;gBAAE,MAAK;YAEnB,OAAO,EAAE,CAAA;QACX,CAAC,QAAQ,OAAO,GAAG,qBAAqB,EAAC;QAEzC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE;aACzC,CAAC,CAAA;QACJ,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,sBAAsB,CAAC;gBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,gBAAgB;gBACzB,gBAAgB,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE;aACnE,CAAC,CAAA;QACJ,CAAC;QAED,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;QAEzD,OAAO,OAAO,CAAC,MAAM,EAAE,CAAA;IACzB,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,OAAsD,EACtD,OAAmC,EACnC,cAA8B;QAE9B,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,OAAO,EAAE,cAAc,CAAC,CAAA;YACtC,OAAO,IAAI,CAAA;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE;gBACvB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,OAAO,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;aAC9C,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,KAAK,CAAA;IACd,CAAC;IAEO,oBAAoB,CAAC,OAAgD;QAC3E,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QAE1D,IAAI,IAAI,GAAG,SAAS,OAAO,CAAC,KAAK,EAAE,CAAA;QACnC,IAAI,WAAW,EAAE,IAAI,EAAE,CAAC,MAAM;YAAE,IAAI,IAAI,IAAI,WAAW,CAAC,IAAI,EAAE,EAAE,CAAA;QAEhE,OAAO,IAAI,CAAA;IACb,CAAC;IAEO,iBAAiB,CAAC,OAAgD;QACxE,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,2BAA2B,EAAE,CAAC,CAAA;QACnE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,KAAK,GAAG,UAAU,EAAE,CAAA;QAE7D,OAAO;YACL,KAAK;YACL,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACxB,cAAc,EAAE,KAAK;gBACrB,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,UAAU,EAAE,OAAO,CAAC,GAAG;aACxB,CAAC;SACH,CAAA;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const safeJsonDeserializer: (data?: string | Buffer) => object | undefined;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export const safeJsonDeserializer = (data) => {
|
|
2
|
+
if (!data)
|
|
3
|
+
return undefined;
|
|
4
|
+
// Checking to be safe
|
|
5
|
+
if (!Buffer.isBuffer(data) && typeof data !== 'string')
|
|
6
|
+
return undefined;
|
|
7
|
+
try {
|
|
8
|
+
return JSON.parse(data.toString('utf-8'));
|
|
9
|
+
}
|
|
10
|
+
catch (_) {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
//# sourceMappingURL=safeJsonDeserializer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"safeJsonDeserializer.js","sourceRoot":"","sources":["../../lib/utils/safeJsonDeserializer.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,IAAsB,EAAsB,EAAE;IACjF,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAA;IAC3B,sBAAsB;IACtB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAA;IAExE,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAA;IAC3C,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC,CAAA"}
|