@lucaapp/service-utils 1.4.0 → 1.5.0

package/README.md

@@ -0,0 +1,41 @@
+## service-utils
+
+`service-utils` aims at swapping common functionality for most/all backend services in order to reduce code duplication.
+Future use-cases should include logging, inter-service communication, metrics etc.
+
+#### Development
+
+Development of service-utils package will take place in your local machine, not within the docker context.
+Therefore, some adoptions and preparations are necessary.
+Follow these steps for initial setup:
+
+* Navigate to `packages/service-utils`
+* Run `yarn link`
+* Navigate to `services/backend`
+* Run `yarn link @lucaapp/service-utils`
+
+This will connect your local sources for `@lucaapp/service-utils`.
+When making changes within the package, make sure to run the `build` target within `packages/service-utils/package.json` to have the changes available to backend web.
+
+After this preparations run `package-development` from `services/backend/package.json`.
+The following environment should be at least in place when running the target to make sure your local `backend` setup uses the proper endpoints:
+
+```shell
+DB_HOSTNAME=localhost
+KAFKA_BROKER=localhost:9094
+REDIS_HOSTNAME=localhost
+```
+
+It will utilize the linked package and run `backend` locally.
+Make sure your docker stack has at least `database`, `kafka` and `redis`, but *not* `backend` up and running.
+
+You should now be able to develop changes for `@lucaapp/service-utils`.
+
+#### Packaging
+
+Packaging of `@lucaapp/service-utils` will be automatically published, when run in `dev`.
+See `./gitlab-ci/publish[.template].yml` for reference.
+
+Make sure to follow [semantic-release](https://www.npmjs.com/package/semantic-release) conventions when writing commit messages.
+
+Depending on the used semantic-release indicators a new version of `@lucaapp/service-utils` will be made available and tagged within the luca-web repository.

package/dist/index.d.ts

@@ -1,3 +1,4 @@
 export * from './lib/kafka';
 export * from './lib/serviceIdentity';
 export * from './lib/metrics';
+export * from './lib/wsEvent';

package/dist/index.js

@@ -17,3 +17,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./lib/kafka"), exports);
 __exportStar(require("./lib/serviceIdentity"), exports);
 __exportStar(require("./lib/metrics"), exports);
+__exportStar(require("./lib/wsEvent"), exports);

package/dist/lib/kafka/events/wsEvent.d.ts

@@ -0,0 +1,5 @@
+declare type WsEvent = {
+    subId: string;
+    data: any;
+};
+export type { WsEvent };

package/dist/lib/kafka/events/wsEvent.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/lib/kafka/events.d.ts

@@ -1,21 +1,31 @@
 import type { Payment } from './events/payment';
 import type { Consumer } from './events/consumer';
 import type { Operator } from './events/operator';
+import type { WsEvent } from './events/wsEvent';
 import { Service } from '../serviceIdentity';
 declare enum KafkaTopic {
-    PAYMENTS = "PAYMENTS",
-    CONSUMERS = "CONSUMERS",
-    OPERATORS = "OPERATORS"
+    PAYMENTS = "payments",
+    CONSUMERS = "consumers",
+    OPERATORS = "operators",
+    WS_EVENT_backend = "wsevent_backend",
+    WS_EVENT_backend_pay = "wsevent_backend-pay",
+    WS_EVENT_backend_pos = "wsevent_backend-pos"
 }
 declare type MessageFormats = {
     [KafkaTopic.PAYMENTS]: Payment;
     [KafkaTopic.CONSUMERS]: Consumer;
     [KafkaTopic.OPERATORS]: Operator;
+    [KafkaTopic.WS_EVENT_backend]: WsEvent;
+    [KafkaTopic.WS_EVENT_backend_pay]: WsEvent;
+    [KafkaTopic.WS_EVENT_backend_pos]: WsEvent;
 };
 declare const MessageIssuer: {
-    PAYMENTS: Service;
-    CONSUMERS: Service;
-    OPERATORS: Service;
+    payments: Service;
+    consumers: Service;
+    operators: Service;
+    wsevent_backend: Service;
+    "wsevent_backend-pay": Service;
+    "wsevent_backend-pos": Service;
 };
 export { KafkaTopic, MessageIssuer };
 export type { MessageFormats };

package/dist/lib/kafka/events.js

@@ -4,14 +4,20 @@ exports.MessageIssuer = exports.KafkaTopic = void 0;
 const serviceIdentity_1 = require("../serviceIdentity");
 var KafkaTopic;
 (function (KafkaTopic) {
-    KafkaTopic["PAYMENTS"] = "PAYMENTS";
-    KafkaTopic["CONSUMERS"] = "CONSUMERS";
-    KafkaTopic["OPERATORS"] = "OPERATORS";
+    KafkaTopic["PAYMENTS"] = "payments";
+    KafkaTopic["CONSUMERS"] = "consumers";
+    KafkaTopic["OPERATORS"] = "operators";
+    KafkaTopic["WS_EVENT_backend"] = "wsevent_backend";
+    KafkaTopic["WS_EVENT_backend_pay"] = "wsevent_backend-pay";
+    KafkaTopic["WS_EVENT_backend_pos"] = "wsevent_backend-pos";
 })(KafkaTopic || (KafkaTopic = {}));
 exports.KafkaTopic = KafkaTopic;
 const MessageIssuer = {
     [KafkaTopic.PAYMENTS]: serviceIdentity_1.Service.BACKEND_PAY,
     [KafkaTopic.CONSUMERS]: serviceIdentity_1.Service.BACKEND,
     [KafkaTopic.OPERATORS]: serviceIdentity_1.Service.BACKEND,
+    [KafkaTopic.WS_EVENT_backend]: serviceIdentity_1.Service.PUBSUB,
+    [KafkaTopic.WS_EVENT_backend_pay]: serviceIdentity_1.Service.PUBSUB,
+    [KafkaTopic.WS_EVENT_backend_pos]: serviceIdentity_1.Service.PUBSUB,
 };
 exports.MessageIssuer = MessageIssuer;

package/dist/lib/kafka/kafkaClient.d.ts

@@ -1,7 +1,8 @@
 import { Consumer } from 'kafkajs';
 import { Logger } from 'pino';
 import { ServiceIdentity } from '../serviceIdentity';
-import type { EventPayloadHandler, KafkaConfiguration, KafkaEvent, KafkaTopics, KafkaTopicType } from './types';
+import type { EventPayloadHandler, KafkaConfiguration, KafkaEvent } from './types';
+import { KafkaTopic } from './events';
 declare class KafkaClient {
     private readonly environment;
     private readonly kafkaClient;
@@ -9,8 +10,8 @@ declare class KafkaClient {
     private readonly topicSecrets;
     private readonly admin;
     private readonly producer;
-    private readonly serviceIdentity;
-    constructor(parentLogger: Logger, kafkaConfig: KafkaConfiguration, topicSecrets: Record<KafkaTopics, string | null>, serviceIdentity: ServiceIdentity);
+    readonly serviceIdentity: ServiceIdentity;
+    constructor(parentLogger: Logger, kafkaConfig: KafkaConfiguration, topicSecrets: Record<KafkaTopic, string | null>, serviceIdentity: ServiceIdentity);
     connect: () => Promise<void>;
     private getTopic;
     private getTopicSecret;
@@ -20,7 +21,7 @@ declare class KafkaClient {
     private verifySignature;
     private parseValue;
     private ensureTopics;
-    consume: <T extends KafkaTopicType>(groupId: string, kafkaTopic: T, handler: EventPayloadHandler<T>, fromBeginning?: boolean) => Promise<Consumer>;
-    produce: <T extends KafkaTopicType>(kafkaTopic: T, key: string, value: KafkaEvent<T>) => Promise<void>;
+    consume: <T extends KafkaTopic>(groupId: string, kafkaTopic: T, handler: EventPayloadHandler<T>, fromBeginning?: boolean) => Promise<Consumer>;
+    produce: <T extends KafkaTopic>(kafkaTopic: T, key: string, value: KafkaEvent<T>) => Promise<void>;
 }
 export { KafkaClient };

package/dist/lib/kafka/kafkaClient.js

@@ -38,11 +38,21 @@ const messageProducedSizeCounter = new metrics_1.metricsClient.Histogram({
     labelNames: ['topic'],
     buckets: [64, 128, 256, 512, 1024, 2048, 4096, 8182],
 });
+const messageProduceError = new metrics_1.metricsClient.Counter({
+    name: 'kafka_message_produce_error_count',
+    help: 'Total number of errors during message produce',
+    labelNames: ['topic'],
+});
 const messageConsumedCounter = new metrics_1.metricsClient.Counter({
     name: 'kafka_message_consume_count',
     help: 'Total number of messages consumed',
     labelNames: ['topic', 'groupId'],
 });
+const messageConsumedErrorCounter = new metrics_1.metricsClient.Counter({
+    name: 'kafka_message_consume_error_count',
+    help: 'Total number of errors during message consume',
+    labelNames: ['topic', 'groupId'],
+});
 const messageAcknowledgedCounter = new metrics_1.metricsClient.Counter({
     name: 'kafka_message_consume_ack_count',
     help: 'Total number of messages acknowledged',
@@ -153,17 +163,24 @@ class KafkaClient {
                 await consumer.run({
                     autoCommit: true,
                     eachMessage: async ({ message }) => {
-                        messageConsumedCounter
-                            .labels({ topic: kafkaTopic.valueOf(), groupId })
-                            .inc();
-                        const decryptedValue = await this.decryptValue(kafkaTopic, message.value);
-                        await this.verifySignature(kafkaTopic, decryptedValue, message.headers);
-                        const value = this.parseValue(decryptedValue);
-                        messageAcknowledgedCounter
-                            .labels({ topic: kafkaTopic.valueOf(), groupId })
-                            .inc();
-                        this.logger.debug({ key: message.key, value, timestamp: message.timestamp }, 'Record received');
-                        await handler({ ...message, value });
+                        try {
+                            messageConsumedCounter
+                                .labels({ topic: kafkaTopic.valueOf(), groupId })
+                                .inc();
+                            const decryptedValue = await this.decryptValue(kafkaTopic, message.value);
+                            await this.verifySignature(kafkaTopic, decryptedValue, message.headers);
+                            const value = this.parseValue(decryptedValue);
+                            this.logger.debug({ key: message.key, value, timestamp: message.timestamp }, 'Record received');
+                            await handler({ ...message, value });
+                            messageAcknowledgedCounter
+                                .labels({ topic: kafkaTopic.valueOf(), groupId })
+                                .inc();
+                        }
+                        catch (error) {
+                            messageConsumedErrorCounter
+                                .labels({ topic: kafkaTopic.valueOf(), groupId })
+                                .inc();
+                        }
                     },
                 });
                 return consumer;
@@ -195,6 +212,7 @@ class KafkaClient {
                     .observe(Buffer.byteLength(encryptedValue));
             }
             catch (error) {
+                messageProduceError.labels({ topic }).inc();
                 throw (0, utils_1.logAndGetError)(this.logger, `Could not create producer for topic=${topic}`, error);
             }
         };

package/dist/lib/kafka/types.d.ts

@@ -1,9 +1,7 @@
 import { KafkaTopic, MessageFormats } from './events';
 import { Environment } from '../serviceIdentity';
 import { KafkaMessage } from 'kafkajs';
-declare type KafkaTopics = keyof typeof KafkaTopic;
-declare type KafkaTopicType = typeof KafkaTopic[KafkaTopics];
-declare type KafkaEvent<T extends KafkaTopicType> = {
+declare type KafkaEvent<T extends KafkaTopic> = {
     id: string;
     type: 'create' | 'update' | 'destroy';
     entity: MessageFormats[T];
@@ -16,7 +14,7 @@ declare type KafkaConfiguration = {
     password?: string;
     ssl?: boolean;
 };
-declare type EventPayloadHandler<T extends KafkaTopicType> = (message: Omit<KafkaMessage, 'value'> & {
+declare type EventPayloadHandler<T extends KafkaTopic> = (message: Omit<KafkaMessage, 'value'> & {
     value: KafkaEvent<T>;
 }) => Promise<void>;
-export type { KafkaTopics, KafkaTopicType, KafkaEvent, KafkaConfiguration, EventPayloadHandler, };
+export type { KafkaEvent, KafkaConfiguration, EventPayloadHandler, };

package/dist/lib/serviceIdentity/service.d.ts

@@ -1,6 +1,7 @@
 declare enum Service {
     BACKEND = "backend",
     BACKEND_PAY = "backend-pay",
-    BACKEND_POS = "backend-pos"
+    BACKEND_POS = "backend-pos",
+    PUBSUB = "pubsub"
 }
 export { Service };

package/dist/lib/serviceIdentity/service.js

@@ -6,5 +6,6 @@ var Service;
     Service["BACKEND"] = "backend";
     Service["BACKEND_PAY"] = "backend-pay";
     Service["BACKEND_POS"] = "backend-pos";
+    Service["PUBSUB"] = "pubsub";
 })(Service || (Service = {}));
 exports.Service = Service;

package/dist/lib/wsEvent/index.d.ts

@@ -0,0 +1,4 @@
+import { KafkaClient } from '../kafka';
+declare const generateSubscriptionId: () => string;
+declare const emitWebsocketEvent: (kafkaClient: KafkaClient, subId: string, data: any) => Promise<void>;
+export { generateSubscriptionId, emitWebsocketEvent };

package/dist/lib/wsEvent/index.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.emitWebsocketEvent = exports.generateSubscriptionId = void 0;
+const uuid_1 = require("uuid");
+const crypto_1 = require("crypto");
+const generateSubscriptionId = () => {
+    return (0, crypto_1.randomBytes)(32).toString('base64');
+};
+exports.generateSubscriptionId = generateSubscriptionId;
+const emitWebsocketEvent = async (kafkaClient, subId, data) => {
+    const issuer = kafkaClient.serviceIdentity.identityName;
+    // using WS_EVENT_backend since all WS_EVENTs use same schema
+    const topic = `wsevent_${issuer}`;
+    const message = {
+        id: (0, uuid_1.v4)(),
+        type: 'create',
+        entity: {
+            subId,
+            data,
+        },
+    };
+    await kafkaClient.produce(topic, subId, message);
+};
+exports.emitWebsocketEvent = emitWebsocketEvent;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucaapp/service-utils",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
@@ -21,6 +21,7 @@
     "@types/express": "4.17.13",
     "@types/jest": "^28.1.5",
     "@types/pino": "^7.0.5",
+    "@types/uuid": "^8.3.4",
     "@typescript-eslint/eslint-plugin": "^5.30.6",
     "@typescript-eslint/parser": "^5.30.6",
     "conventional-changelog-conventionalcommits": "^5.0.0",
@@ -40,6 +41,7 @@
     "jose": "4.9.2",
     "kafkajs": "2.1.0",
     "moment": "^2.29.4",
-    "prom-client": "14.1.0"
+    "prom-client": "14.1.0",
+    "uuid": "^9.0.0"
   }
 }