@palmetto/pubsub 2.0.0 → 2.1.1

package/README.md

@@ -49,6 +49,7 @@ yarn add @palmetto/pubsub zod
      schema: TestModelSchema,
      transport: RABBITMQ_TRANSPORT,
      exchangeType: "topic",
+     messageLogLevel: "info",
    };
 
    const publisher = new Publisher(console, [rabbitMqPublisher]);
@@ -92,6 +93,7 @@ yarn add @palmetto/pubsub zod
      retries: 20,
      transport: RABBITMQ_TRANSPORT,
      exchangeType: "fanout",
+     messageLogLevel: "debug",
    };
 
    function onMessage(message: Model) {
@@ -106,6 +108,18 @@ yarn add @palmetto/pubsub zod
    await subscriber.startSubscribe(schemaConfig, onMessage);
    ```
 
+### Message logging
+
+For each message you can indicate how to log messages when they are published or handled by a subscriber. Use the `messageLogLevel` property of the queue configuration object.
+
+Valid values are:
+
+- off
+- debug (default)
+- info
+- warn
+- error
+
 ### RabbitMQ Usage
 
 For specific details about using RabbitMQ see [RabbitMQ README.md](src/rabbitmq/README.md)

package/dist/bullmq/config.d.ts

@@ -1,5 +1,10 @@
 import { MessageContext, PubSubConfiguration } from "../interfaces.js";
+import { BULLMQ_TRANSPORT } from "./connection.js";
 export interface BullMqQueueConfiguration extends PubSubConfiguration {
+    /**
+     * Transport is always BULLMQ_TRANSPORT
+     */
+    transport: typeof BULLMQ_TRANSPORT;
     /**
      * The name of the queue
      */
@@ -19,3 +24,4 @@ export interface BullMqMessageContext extends MessageContext {
      */
     reply?: unknown;
 }
+export declare function isBullMqQueueConfiguration(config: PubSubConfiguration): config is BullMqQueueConfiguration;

package/dist/bullmq/config.js

@@ -1,2 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.isBullMqQueueConfiguration = isBullMqQueueConfiguration;
+const connection_js_1 = require("./connection.js");
+function isBullMqQueueConfiguration(config) {
+    return config.transport === connection_js_1.BULLMQ_TRANSPORT;
+}

package/dist/bullmq/publisher.d.ts

@@ -11,4 +11,5 @@ export declare class BullMqPublisher implements PublisherProvider {
     publish(config: BullMqQueueConfiguration, message: string): Promise<void>;
     init(config: BullMqQueueConfiguration): Promise<void>;
     close(): Promise<void>;
+    enrichPublishedMesssageLog(config: BullMqQueueConfiguration): Record<string, unknown>;
 }

package/dist/bullmq/publisher.js

@@ -63,5 +63,10 @@ class BullMqPublisher {
             this.queues.clear();
         });
     }
+    enrichPublishedMesssageLog(config) {
+        return {
+            job: config.job,
+        };
+    }
 }
 exports.BullMqPublisher = BullMqPublisher;

package/dist/bullmq/pubsub.d.ts

@@ -10,4 +10,6 @@ export declare class BullMqPubSubProvider implements PubSubProvider {
     startSubscribe(config: BullMqQueueConfiguration, onMessage: (s: string, context: MessageContext) => Promise<MessageResult> | MessageResult): Promise<StopSubscribe>;
     close(): Promise<void>;
     init(config: BullMqQueueConfiguration): Promise<void>;
+    enrichPublishedMesssageLog(config: BullMqQueueConfiguration): Record<string, unknown>;
+    enrichHandledMesssageLog(config: BullMqQueueConfiguration): Record<string, unknown>;
 }

package/dist/bullmq/pubsub.js

@@ -36,5 +36,11 @@ class BullMqPubSubProvider {
             yield this.publisher.init(config);
         });
     }
+    enrichPublishedMesssageLog(config) {
+        return this.publisher.enrichPublishedMesssageLog(config);
+    }
+    enrichHandledMesssageLog(config) {
+        return this.subscriber.enrichHandledMesssageLog(config);
+    }
 }
 exports.BullMqPubSubProvider = BullMqPubSubProvider;

package/dist/bullmq/subscriber.d.ts

@@ -23,5 +23,6 @@ export declare class BullMqSubscriber implements SubscriberProvider {
     removeSubscriber(subscribedMessage: SubscribedMessage): void;
     startSubscribe(config: BullMqQueueConfiguration, onMessage: (s: string, context: BullMqMessageContext) => Promise<MessageResult> | MessageResult): Promise<StopSubscribe>;
     close(): Promise<void>;
+    enrichHandledMesssageLog(config: BullMqQueueConfiguration): Record<string, unknown>;
 }
 export {};

package/dist/bullmq/subscriber.js

@@ -118,5 +118,10 @@ class BullMqSubscriber {
             this.stops.clear();
         });
     }
+    enrichHandledMesssageLog(config) {
+        return {
+            job: config.job,
+        };
+    }
 }
 exports.BullMqSubscriber = BullMqSubscriber;

package/dist/create-log-error-payload.d.ts

@@ -1,8 +1,2 @@
-export interface LogErrorPayload {
-    message: string;
-    kind: string;
-    stack?: string;
-    cause?: unknown;
-    statusCode?: number;
-}
+import { LogErrorPayload } from "./log";
 export declare function createLogErrorPayload(error: unknown): LogErrorPayload;

package/dist/create-log-error-payload.js

@@ -1,5 +1,5 @@
 "use strict";
-/* compare to code in @palmetto/nestjs-logger */
+/* copied from @palmetto/nestjs-logger */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createLogErrorPayload = createLogErrorPayload;
 function createLogErrorPayload(error) {

package/dist/interfaces.d.ts

@@ -25,7 +25,9 @@ export interface PublisherProvider {
     init(config: PubSubConfiguration): Promise<void> | void;
     publish(config: PubSubConfiguration, message: string): Promise<void> | void;
     close(): Promise<void> | void;
+    enrichPublishedMesssageLog(config: PubSubConfiguration): Record<string, unknown>;
 }
+export type MessageLogLevel = "error" | "warn" | "info" | "debug" | "off";
 /**
  * Minimal configuration needed to publish or subscribe to a message
  */
@@ -50,6 +52,10 @@ export interface PubSubConfiguration {
      * Minimum amount of time to wait, in milliseconds, before retrying a message
      */
     retryDelay?: number;
+    /**
+     * log level to use when logging message payloads (default: debug)
+     */
+    messageLogLevel?: MessageLogLevel;
 }
 export declare enum MessageResult {
     /**
@@ -105,6 +111,7 @@ export interface SubscriberProvider {
     transport: string;
     startSubscribe(config: PubSubConfiguration, onMessage: (s: string, context: MessageContext) => Promise<MessageResult> | MessageResult): Promise<StopSubscribe> | StopSubscribe;
     close(): Promise<void> | void;
+    enrichHandledMesssageLog(config: PubSubConfiguration): Record<string, unknown>;
 }
 /**
  * A class that implements both the publisher and subscriber

package/dist/log.d.ts

@@ -0,0 +1,37 @@
+/** copied from @nestjs/logger */
+export type Log<TRequiredKeys extends "http" | "error" | "message" | "extra" = "message"> = Omit<LogPayload, TRequiredKeys> & Required<Pick<LogPayload, TRequiredKeys>>;
+export interface LogPayload {
+    message: string;
+    error?: LogErrorPayload;
+    http?: LogHttpPayload;
+    identity?: LogIdentityPayload;
+    extra?: Record<string, unknown>;
+}
+export interface LogErrorPayload {
+    message: string;
+    kind: string;
+    stack?: string;
+    cause?: unknown;
+    statusCode?: number;
+}
+export interface LogHttpPayload {
+    method: string;
+    status_code: number;
+    url?: string;
+    route?: string;
+    client_ip?: string;
+    useragent?: string;
+    referer?: string;
+    request?: {
+        headers?: Record<string, string>;
+        content_length?: number;
+    };
+    response?: {
+        headers?: Record<string, string>;
+        content_length?: number;
+    };
+}
+export interface LogIdentityPayload {
+    userId: string;
+    organizationId: string;
+}

package/dist/log.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/message-logger.d.ts

@@ -0,0 +1,10 @@
+import { Logger, MessageLogLevel } from "./interfaces";
+export declare function startTiming(): [number, number];
+export declare function getDuration(start: [number, number]): number;
+export declare function logMessage({ note, message, logger, level, extra, }: {
+    note: string;
+    message: unknown;
+    logger: Logger;
+    level?: MessageLogLevel;
+    extra?: Record<string, unknown>;
+}): void;

package/dist/message-logger.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startTiming = startTiming;
+exports.getDuration = getDuration;
+exports.logMessage = logMessage;
+function startTiming() {
+    return process.hrtime();
+}
+function getDuration(start) {
+    return process.hrtime(start)[1] / 1000000; // convert to milliseconds
+}
+function logMessage({ note, message, logger, level = "debug", extra, }) {
+    var _a;
+    if (level === "off") {
+        return;
+    }
+    const logFn = level === "info" ? logger.log.bind(logger) : (_a = logger.debug) === null || _a === void 0 ? void 0 : _a.bind(logger);
+    if (!logFn) {
+        return;
+    }
+    const msg = {
+        message: note,
+        extra: Object.assign(Object.assign({}, extra), { message }),
+    };
+    logFn(msg);
+}

package/dist/publisher.js

@@ -14,6 +14,7 @@ const zod_1 = require("zod");
 const uuid_1 = require("uuid");
 const crypto_1 = require("crypto");
 const errors_js_1 = require("./errors.js");
+const message_logger_js_1 = require("./message-logger.js");
 class Publisher {
     constructor(logger, providers) {
         var _a, _b;
@@ -52,7 +53,6 @@ class Publisher {
     }
     publish(config, message) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
             const provider = this.getProvider(config);
             if (!message.id) {
                 message.id = (0, uuid_1.v4)();
@@ -70,11 +70,26 @@ class Publisher {
             }
             const check = schema.safeEncode(message);
             if (!check.success) {
+                (0, message_logger_js_1.logMessage)({
+                    note: "Publish message failed schema validation",
+                    message: message,
+                    level: "error",
+                    logger: this.logger,
+                    extra: Object.assign({ transport: provider.transport, name: config.name }, provider.enrichPublishedMesssageLog(config)),
+                });
                 throw new errors_js_1.SchemaValidationError(`Schema did not accept the published message: ${check.error.message}`);
             }
-            (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `Publisher publishing message for ${provider.transport}`);
             const json = JSON.stringify(check.data);
+            const start = (0, message_logger_js_1.startTiming)();
             yield provider.publish(config, json);
+            const duration = (0, message_logger_js_1.getDuration)(start);
+            (0, message_logger_js_1.logMessage)({
+                note: "Published message",
+                message: check.data,
+                level: config.messageLogLevel,
+                logger: this.logger,
+                extra: Object.assign({ transport: provider.transport, name: config.name, durationMs: duration }, provider.enrichPublishedMesssageLog(config)),
+            });
         });
     }
     close() {

package/dist/rabbitmq/config.d.ts

@@ -1,5 +1,6 @@
 import { AmqpConnectionManagerOptions } from "amqp-connection-manager";
 import { MessageContext, PubSubConfiguration } from "../interfaces.js";
+import { RABBITMQ_TRANSPORT } from "./connection.js";
 export interface RabbitMqConnectionConfig {
     /**
      * The URL to the RabbitMQ host
@@ -21,6 +22,10 @@ export interface RabbitMqConnectionConfig {
 export type QueueType = "default" | "dead-letter" | "retry";
 export type ExchangeType = "direct" | "topic" | "fanout";
 export interface RabbitQueueExchangeConfiguration extends PubSubConfiguration {
+    /**
+     * Transport is always RABBITMQ_TRANSPORT
+     */
+    transport: typeof RABBITMQ_TRANSPORT;
     /**
      * The queue name prefix
      */
@@ -56,6 +61,7 @@ export interface RabbitQueueExchangeConfiguration extends PubSubConfiguration {
      */
     publishToSpecificQueue?: string;
 }
+export declare function isRabbitQueueExchangeConfiguration(config: PubSubConfiguration): config is RabbitQueueExchangeConfiguration;
 export interface RabbitQueueExchangeNames {
     queueName?: string;
     exchangeName?: string;

package/dist/rabbitmq/config.js

@@ -1,12 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.QueueNameExtensions = exports.ExchangeNameExtensions = void 0;
+exports.isRabbitQueueExchangeConfiguration = isRabbitQueueExchangeConfiguration;
 exports.getQueueName = getQueueName;
 exports.getExchangeName = getExchangeName;
 exports.getQueueType = getQueueType;
 exports.getRoutingKey = getRoutingKey;
 exports.getExchangeType = getExchangeType;
 const errors_js_1 = require("../errors.js");
+const connection_js_1 = require("./connection.js");
+function isRabbitQueueExchangeConfiguration(config) {
+    return config.transport === connection_js_1.RABBITMQ_TRANSPORT;
+}
 /**
  * Returns the queue name based on the configuration
  *

package/dist/rabbitmq/publisher.d.ts

@@ -25,6 +25,7 @@ export declare class RabbitMqPublisher implements PublisherProvider {
      * @returns A promise that is completed when the message is published
      */
     publish(config: RabbitQueueExchangeConfiguration, message: string): Promise<void>;
+    enrichPublishedMesssageLog(config: RabbitQueueExchangeConfiguration): Record<string, unknown>;
     private publishToExchange;
     private publishToQueue;
     close(): Promise<void> | undefined;

package/dist/rabbitmq/publisher.js

@@ -89,17 +89,25 @@ class RabbitMqPublisher {
             }
         });
     }
+    enrichPublishedMesssageLog(config) {
+        if (config.publishToSpecificQueue) {
+            return {
+                publishToSpecificQueue: config.publishToSpecificQueue,
+            };
+        }
+        return {
+            exchangeName: (0, config_js_1.getExchangeName)(config),
+            routingKey: (0, config_js_1.getRoutingKey)(config),
+        };
+    }
     publishToExchange(channel, config, message) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d;
             const exchangeName = (0, config_js_1.getExchangeName)(config);
-            (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `Publishing message to ${exchangeName} - ${message} [starting]`);
             const ok = yield channel.publish(exchangeName, (0, config_js_1.getRoutingKey)(config), Buffer.from(message, "utf8"), {
                 contentType: "application/json",
                 timestamp: new Date().valueOf(),
                 persistent: true,
             });
-            (_d = (_c = this.logger).debug) === null || _d === void 0 ? void 0 : _d.call(_c, `Published message to ${exchangeName} - ${message} [${ok}]`);
             if (!ok) {
                 throw new errors_js_1.PublishError(`RabbitMq publish to ${exchangeName} failed`);
             }
@@ -107,14 +115,11 @@ class RabbitMqPublisher {
     }
     publishToQueue(channel, queue, message) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d;
-            (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `Publishing message to queue:${queue} - ${message} [starting]`);
             const ok = yield channel.sendToQueue(queue, Buffer.from(message, "utf8"), {
                 contentType: "application/json",
                 timestamp: new Date().valueOf(),
                 persistent: true,
             });
-            (_d = (_c = this.logger).debug) === null || _d === void 0 ? void 0 : _d.call(_c, `Published message to queue:${queue} - ${message} [${ok}]`);
             if (!ok) {
                 throw new errors_js_1.PublishError(`RabbitMq publish to queue:${queue} failed`);
             }

package/dist/rabbitmq/pubsub.d.ts

@@ -13,4 +13,6 @@ export declare class RabbitMqPubSubProvider implements PubSubProvider {
     startSubscribe(config: RabbitQueueExchangeConfiguration, onMessage: (s: string, context: MessageContext) => Promise<MessageResult> | MessageResult): StopSubscribe;
     close(): Promise<void>;
     init(config: RabbitQueueExchangeConfiguration): Promise<void>;
+    enrichPublishedMesssageLog(config: RabbitQueueExchangeConfiguration): Record<string, unknown>;
+    enrichHandledMesssageLog(config: RabbitQueueExchangeConfiguration): Record<string, unknown>;
 }

package/dist/rabbitmq/pubsub.js

@@ -38,5 +38,11 @@ class RabbitMqPubSubProvider {
             yield this.publisher.init(config);
         });
     }
+    enrichPublishedMesssageLog(config) {
+        return this.publisher.enrichPublishedMesssageLog(config);
+    }
+    enrichHandledMesssageLog(config) {
+        return this.subscriber.enrichHandledMesssageLog(config);
+    }
 }
 exports.RabbitMqPubSubProvider = RabbitMqPubSubProvider;

package/dist/rabbitmq/subscriber.d.ts

@@ -23,6 +23,7 @@ export declare class RabbitMqSubscriber implements SubscriberProvider {
     removeSubscriber(subscribedMessage: SubscribedMessage): void;
     startSubscribe(config: RabbitQueueExchangeConfiguration, onMessage: (s: string, context: RabbitMqMessageContext) => Promise<MessageResult> | MessageResult): StopSubscribe;
     close(): Promise<void>;
+    enrichHandledMesssageLog(config: RabbitQueueExchangeConfiguration): Record<string, unknown>;
     static getRetries(msg: ConsumeMessage): number;
     static getSentDates(msg: ConsumeMessage): {
         firstSent: Date | undefined;

package/dist/rabbitmq/subscriber.js

@@ -89,6 +89,11 @@ class RabbitMqSubscriber {
             this.subscribers.clear();
         });
     }
+    enrichHandledMesssageLog(config) {
+        return {
+            queueName: (0, config_js_1.getQueueName)(config),
+        };
+    }
     static getRetries(msg) {
         var _a;
         return typeof ((_a = msg.properties.headers) === null || _a === void 0 ? void 0 : _a[RETRIES_HEADER]) === "number"

package/dist/subscriber.js

@@ -13,6 +13,8 @@ exports.Subscriber = void 0;
 const interfaces_js_1 = require("./interfaces.js");
 const errors_js_1 = require("./errors.js");
 const events_1 = require("events");
+const message_logger_js_1 = require("./message-logger.js");
+const create_log_error_payload_js_1 = require("./create-log-error-payload.js");
 class SubscribedMessage {
     constructor(stop) {
         this.stop = stop;
@@ -54,23 +56,23 @@ class Subscriber {
                 throw new errors_js_1.MissingPubSubProviderError(`No provider configured for ${config.transport}`);
             }
             const transform = (jsonStr, context) => __awaiter(this, void 0, void 0, function* () {
-                var _a;
-                const start = new Date().valueOf();
+                const start = (0, message_logger_js_1.startTiming)();
                 const jsonObject = JSON.parse(jsonStr);
                 const r = schema.safeDecode(jsonObject);
                 if (!r.success) {
-                    const id = interfaces_js_1.IdMetaSchema.safeParse(jsonObject);
-                    if (id.success) {
-                        this.logger.warn(`Unable to parse message ${id.data.id} created at ${(_a = id.data.meta) === null || _a === void 0 ? void 0 : _a.createdAt} for ${config.transport}:${config.name} - schemaError`);
-                    }
-                    else {
-                        this.logger.warn(`Unable to parse message for ${config.transport}:${config.name} - schemaError`);
-                    }
+                    const durationMs = (0, message_logger_js_1.getDuration)(start);
+                    (0, message_logger_js_1.logMessage)({
+                        note: "Subscribed message failed schema validation",
+                        message: jsonObject,
+                        level: "error",
+                        logger: this.logger,
+                        extra: Object.assign({ transport: provider.transport, name: config.name, durationMs }, provider.enrichHandledMesssageLog(config)),
+                    });
                     const handledEventContext = {
                         message: jsonStr,
                         context,
                         config,
-                        durationMs: new Date().valueOf() - start,
+                        durationMs,
                         result: interfaces_js_1.MessageResult.Fail,
                     };
                     this.events.emit("schemaError", handledEventContext);
@@ -79,22 +81,39 @@ class Subscriber {
                 }
                 try {
                     const result = yield onMessage(r.data, context);
+                    const durationMs = (0, message_logger_js_1.getDuration)(start);
                     const eventContext = {
                         message: jsonStr,
                         context,
                         config,
-                        durationMs: new Date().valueOf() - start,
+                        durationMs,
                         result,
                     };
+                    (0, message_logger_js_1.logMessage)({
+                        note: "Subscriber processed message",
+                        message: r.data,
+                        level: config.messageLogLevel,
+                        logger: this.logger,
+                        extra: Object.assign({ transport: provider.transport, name: config.name, durationMs,
+                            result }, provider.enrichHandledMesssageLog(config)),
+                    });
                     this.events.emit("messageHandled", eventContext);
                     return result;
                 }
                 catch (err) {
+                    const durationMs = (0, message_logger_js_1.getDuration)(start);
+                    (0, message_logger_js_1.logMessage)({
+                        note: "Subscriber error when processing message",
+                        message: r.data,
+                        level: "error",
+                        logger: this.logger,
+                        extra: Object.assign(Object.assign({ transport: provider.transport, name: config.name, durationMs }, provider.enrichHandledMesssageLog(config)), { error: (0, create_log_error_payload_js_1.createLogErrorPayload)(err) }),
+                    });
                     const eventContext = {
                         message: jsonStr,
                         context,
                         config,
-                        durationMs: new Date().valueOf() - start,
+                        durationMs,
                         err,
                     };
                     this.events.emit("messageHandled", eventContext);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@palmetto/pubsub",
-  "version": "2.0.0",
+  "version": "2.1.1",
   "main": "./dist/main.js",
   "scripts": {
     "lint": "yarn run -T eslint --fix ./src",