@palmetto/pubsub 3.3.2 → 3.4.1

package/README.md

@@ -18,6 +18,8 @@ It is up to the pub/sub implementations what to do with rejected messages. Optio
 
 ## Installation
 
+Install pubsub and the basic peer dependencies. You'll also need to install a pubsub system like RabbitMq or BullMq. See below for links on how to use each supported pubsub system.
+
 ```sh
 yarn add @palmetto/pubsub zod @palmetto/trace dd-trace
 ```
@@ -189,6 +191,7 @@ The message is checked against the provided schema for every publish and subscri
       message: string, // the JSON string of the message
       context: MessageContext, // the context of the message provided by the SubscriptionProvider
       config: PubSubConfiguration, // the configuration for the message (eg: BullMqQueueConfiguration, RabbitQueueExchangeConfiguration)
+      error: z.ZodError, // the actual zod error from the schema
     ) => {
       // TODO
     },
@@ -209,3 +212,76 @@ It is up to the provider to obey the `retries` and `retryDelay` configuration pr
 ### Publisher errors
 
 When a message publish fails, it is up to the calling code to handle exceptions.
+
+## Events
+
+Publisher and Subscribers fire events you can use to inspect messages.
+
+### Publisher events
+
+- schemaError: fired when the message fails zod validation
+- messagePublishing: fired before the message is published to the pubsub provider
+- messagePublished: fired after the message was published via the pubsub provider
+
+```ts
+import {
+  PublishingEventContext,
+  PublisherSchemaErrorEventContext,
+  PublishedEventContext,
+} from "@palmetto/pubsub";
+
+import { z } from "zod";
+
+// when a schema validation fails none of the messages in a published batch are sent
+publisher.on("schemaError", (ctx: PublisherSchemaErrorEventContext) => {
+  console.log(z.prettifyError(ctx.error));
+});
+
+// just before publishing the message
+publisher.on("messagePublishing", (ctx: PublishingEventContext) => {
+  console.log(ctx.json);
+});
+
+// just after publishing the message
+publisher.on("messagePublished", (ctx: PublishedEventContext) => {
+  console.log(ctx.json);
+});
+```
+
+### Subscriber events
+
+- messageReceived: fired when the plain JSON is received by the pubsub provider. You can modify the JSON at this time
+- schemaError: fired when the message fails zod validation.
+- messageHandling: fired before the message was published via the pubsub provider
+- messageHandled: fired after the message was published via the pubsub provider
+
+```ts
+import {
+  MessageReceivedEventContext,
+  SubscriberSchemaErrorEventContext,
+  HandlingEventContext,
+  HandledEventContext,
+} from "@palmetto/pubsub";
+
+import { z } from "zod";
+
+// before any processing of the message
+subscriber.on("messageReceived", (ctx: ReceivedEventContext) => {
+  console.log(ctx.json);
+});
+
+// when a schema validation fails
+subscriber.on("schemaError", (ctx: SubscriberSchemaErrorEventContext) => {
+  console.log(z.prettifyError(ctx.error));
+});
+
+// before calling the message handler
+subscriber.on("messageHandling", (ctx: HandlingEventContext) => {
+  console.log(ctx.json);
+});
+
+// after calling the message handler
+subscriber.on("messageHandled", (ctx: HandledEventContext) => {
+  console.log(ctx.json);
+});
+```

package/dist/publisher.d.ts

@@ -1,9 +1,61 @@
+import { z } from "zod";
 import { BaseMessage, Logger, PublisherProvider, PublishMessageOptions, PubSubConfiguration } from "./interfaces.js";
+export interface BasePublisherEventContext {
+    /**
+     * The original message sent into pubsub before it was parsed by the zod schema.
+     */
+    originalMessage: BaseMessage;
+    /**
+     * The message converted to JSON
+     */
+    json: string;
+    /**
+     * The configuration for the publisher that will publish this message.
+     */
+    config: PubSubConfiguration;
+}
+export interface PublishingEventContext extends BasePublisherEventContext {
+    /**
+     * The message being published. This is the message after it was parsed by the zod schema.
+     */
+    message: unknown;
+}
+export interface PublisherSchemaErrorEventContext extends BasePublisherEventContext {
+    /**
+     * The zod validation error that occurred when validating the message against the schema defined in the PubSubConfiguration.
+     */
+    error: z.ZodError;
+}
+export interface PublishedEventContext extends BasePublisherEventContext {
+    /**
+     * The message that was published. This is the message after it was parsed by the zod schema
+     */
+    message: unknown;
+}
+export interface PublisherEventMap {
+    /**
+     * Fired before a message is published. This can be used to log the published message or to perform additional actions before a message is published.
+     * @param context The context of the published message
+     */
+    messagePublishing: [context: PublishingEventContext];
+    /**
+     * Fired when a message fails validation. This can be used to log the validation error or to perform additional actions when a message fails validation.
+     * @param context The context of the validation error
+     */
+    schemaError: [context: PublisherSchemaErrorEventContext];
+    /**
+     * Fired after a message is published. This can be used to log the published message or to perform additional actions after a message is published.
+     * @param context The context of the published message
+     */
+    messagePublished: [context: PublishedEventContext];
+}
 export declare class Publisher {
     private readonly logger;
     private readonly hashes;
     private readonly publisherProviders;
+    private readonly events;
     constructor(logger: Logger, providers?: PublisherProvider[]);
+    on<TK extends keyof PublisherEventMap>(event: TK, listener: (...args: PublisherEventMap[TK]) => void): this;
     addProvider(provider: PublisherProvider): void;
     removeProvider(providerOrTransport: PublisherProvider | string): boolean;
     init(config: PubSubConfiguration): Promise<void>;

package/dist/publisher.js

@@ -16,12 +16,14 @@ const crypto_1 = require("crypto");
 const trace_1 = require("@palmetto/trace");
 const errors_js_1 = require("./errors.js");
 const message_logger_js_1 = require("./message-logger.js");
+const events_1 = require("events");
 class Publisher {
     constructor(logger, providers) {
         var _a, _b;
         this.logger = logger;
         this.hashes = new Map();
         this.publisherProviders = new Map();
+        this.events = new events_1.EventEmitter();
         if (providers) {
             providers.forEach((provider) => this.publisherProviders.set(provider.transport, provider));
         }
@@ -29,6 +31,14 @@ class Publisher {
             ...this.publisherProviders.keys(),
         ].join(", ")}`);
     }
+    on(event, listener) {
+        this.events.on("messagePublished", (context) => {
+            var _a, _b;
+            (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `Event 'messagePublished' emitted for ${context.config.name} on transport ${context.config.transport}`);
+        });
+        this.events.on(event, listener);
+        return this;
+    }
     addProvider(provider) {
         var _a, _b;
         this.publisherProviders.set(provider.transport, provider);
@@ -59,9 +69,7 @@ class Publisher {
         }
         return (0, trace_1.getTracer)().trace("pubsub.publish", {
             resource: `publish ${config.transport} ${config.name}`,
-        }, (span) => {
-            return this.publishImpl(config, messages, options, span);
-        });
+        }, (span) => this.publishImpl(config, messages, options, span));
     }
     publishImpl(config, messages, options, span) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -91,10 +99,17 @@ class Publisher {
                         logger: this.logger,
                         extra: Object.assign({ transport: provider.transport, name: config.name }, provider.enrichPublishedMesssageLog(config)),
                     });
-                    return { json: JSON.stringify(msg), error: check.error };
+                    const json = JSON.stringify(msg);
+                    this.events.emit("schemaError", {
+                        originalMessage: msg,
+                        error: check.error,
+                        json,
+                        config,
+                    });
+                    return { json, msg, error: check.error };
                 }
                 const json = JSON.stringify(check.data);
-                return { json, data: check.data };
+                return { json, msg, data: check.data };
             });
             const errors = mappedMessages.filter((mm) => mm.error);
             if (errors.length > 0) {
@@ -102,9 +117,20 @@ class Publisher {
                 throw new errors_js_1.SchemaValidationError(`Schema did not accept ${errors.length} of ${messages.length} message(s): ${errorMessage}`);
             }
             const jsons = mappedMessages.filter((j) => j.data).map((j) => j.json);
+            if (this.events.listenerCount("messagePublishing") > 0) {
+                mappedMessages.forEach((msg) => {
+                    this.events.emit("messagePublishing", {
+                        message: msg.data,
+                        originalMessage: msg.msg,
+                        json: msg.json,
+                        config,
+                    });
+                });
+            }
             const start = (0, message_logger_js_1.startTiming)();
             yield provider.publish(config, jsons, options);
             const duration = (0, message_logger_js_1.getDuration)(start);
+            const hasPublishedListener = this.events.listenerCount("messagePublished") > 0;
             mappedMessages.forEach((msg) => {
                 (0, message_logger_js_1.logMessage)({
                     note: "Published message",
@@ -113,6 +139,14 @@ class Publisher {
                     logger: this.logger,
                     extra: Object.assign({ transport: provider.transport, name: config.name, durationMs: duration, batchSize: messages.length }, provider.enrichPublishedMesssageLog(config)),
                 });
+                if (hasPublishedListener) {
+                    this.events.emit("messagePublished", {
+                        message: msg.data,
+                        originalMessage: msg.msg,
+                        json: msg.json,
+                        config,
+                    });
+                }
             });
             if (span) {
                 span.setTag("pubsub.duration_ms", duration);

package/dist/subscriber.d.ts

@@ -1,17 +1,59 @@
 import { BaseMessage, Logger, MessageContext, MessageResult, PubSubConfiguration, SubscriberProvider } from "./interfaces.js";
-export interface SubscriberEventContext {
-    message: string;
+import { z } from "zod";
+export interface BaseSubscriberEventContext {
+    /**
+     * The plain JSON message as received from the provider.
+     */
+    json: string;
+    /**
+     * The message context as provided by the provider. This includes metadata about the message such as attemptsMade and willAttemptRetry which can be used to determine if the message is being retried and how many times it has been attempted before.
+     */
     context: MessageContext;
+    /**
+     * The configuration for the subscriber that received this message. This can be used in the event listeners to get information about the subscriber such as the transport and name, and can be used in the enrichHandledMesssageLog to add additional information to the logs.
+     */
     config: PubSubConfiguration;
 }
-export interface MessageHandledEventContext extends SubscriberEventContext {
+export interface ReceivedEventContext extends BaseSubscriberEventContext {
+}
+export interface SubscriberSchemaErrorEventContext extends BaseSubscriberEventContext {
+    /**
+     * The zod validation error that occurred when validating the message against the schema defined in the PubSubConfiguration.
+     */
+    error: z.ZodError;
+}
+export interface HandlingEventContext extends BaseSubscriberEventContext {
+    /**
+     * The message after it has been parsed by the zod schema.
+     */
+    message: unknown;
+}
+export interface HandledEventContext extends HandlingEventContext {
     durationMs: number;
     result?: MessageResult;
     err?: unknown;
 }
-export interface SubscriberEvents {
-    schemaError: (context: SubscriberEventContext) => void;
-    messageHandled: (context: MessageHandledEventContext) => void;
+export interface SubscriberEventMap {
+    /**
+     * Fired when a message is received, before schema validation. This can be used to modify the raw message before validation, for example to add missing properties or to log the raw message.
+     * @param context The context of the received message
+     */
+    messageReceived: [context: ReceivedEventContext];
+    /**
+     * This is fired when a message fails schema validation. Note that the messageHandled event is also fired after this event, so you can use this event to log schema validation errors separately from processing errors in the onMessage callback.
+     * @param context The context of the message that failed schema validation
+     */
+    schemaError: [context: SubscriberSchemaErrorEventContext];
+    /**
+     * Fired before a message is being sent to the onMessage callback.
+     * @param context The context of the message that is being processed
+     */
+    messageHandling: [context: HandlingEventContext];
+    /**
+     * Fired after a message was processed and a result is available, or when an error has been caught in the onMessage callback.
+     * @param context The context of the message that was handled
+     */
+    messageHandled: [context: HandledEventContext];
 }
 export declare class Subscriber {
     private readonly logger;
@@ -19,7 +61,7 @@ export declare class Subscriber {
     private readonly events;
     private readonly subscribedMessages;
     constructor(logger: Logger, providers?: SubscriberProvider[]);
-    on<TU extends keyof SubscriberEvents>(event: TU, listener: SubscriberEvents[TU]): this;
+    on<TU extends keyof SubscriberEventMap>(event: TU, listener: (...args: SubscriberEventMap[TU]) => void): this;
     /**
      * Starts the subscriber for the given configuration. Depending on the protocol, the subscriber may not be started immediately.
      *

package/dist/subscriber.js

@@ -58,6 +58,12 @@ class Subscriber {
             const enrichedConfig = provider.enrichHandledMesssageLog(config);
             const handleMessage = (jsonStr, context) => __awaiter(this, void 0, void 0, function* () {
                 const start = (0, message_logger_js_1.startTiming)();
+                const messageReceivedEventContext = {
+                    json: jsonStr,
+                    context,
+                    config,
+                };
+                this.events.emit("messageReceived", messageReceivedEventContext);
                 const jsonObject = JSON.parse(jsonStr);
                 const decodeResult = schema.safeDecode(jsonObject);
                 if (!decodeResult.success) {
@@ -70,27 +76,37 @@ class Subscriber {
                         extra: Object.assign({ transport: provider.transport, name: config.name, durationMs,
                             context }, enrichedConfig),
                     });
+                    const schemaErrorEventContext = {
+                        json: jsonStr,
+                        context,
+                        config,
+                        error: decodeResult.error,
+                    };
+                    this.events.emit("schemaError", schemaErrorEventContext);
                     const handledEventContext = {
-                        message: jsonStr,
+                        json: jsonStr,
+                        message: undefined,
                         context,
                         config,
                         durationMs,
                         result: interfaces_js_1.MessageResult.Fail,
+                        err: decodeResult.error,
                     };
-                    this.events.emit("schemaError", handledEventContext);
                     this.events.emit("messageHandled", handledEventContext);
                     return interfaces_js_1.MessageResult.Fail;
                 }
                 try {
-                    const result = yield onMessage(decodeResult.data, context);
-                    const durationMs = (0, message_logger_js_1.getDuration)(start);
-                    const eventContext = {
-                        message: jsonStr,
+                    const messageHandlingEventContext = {
+                        json: jsonStr,
+                        message: decodeResult.data,
                         context,
                         config,
-                        durationMs,
-                        result,
                     };
+                    this.events.emit("messageHandling", messageHandlingEventContext);
+                    const result = yield onMessage(decodeResult.data, context);
+                    const durationMs = (0, message_logger_js_1.getDuration)(start);
+                    const eventContext = Object.assign(Object.assign({}, messageHandlingEventContext), { durationMs,
+                        result });
                     (0, message_logger_js_1.logMessage)({
                         note: "Subscriber processed message",
                         message: decodeResult.data,
@@ -118,7 +134,8 @@ class Subscriber {
                         ? interfaces_js_1.MessageResult.Retry
                         : interfaces_js_1.MessageResult.Fail;
                     const eventContext = {
-                        message: jsonStr,
+                        json: jsonStr,
+                        message: decodeResult.data,
                         result,
                         context,
                         config,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@palmetto/pubsub",
-  "version": "3.3.2",
+  "version": "3.4.1",
   "repository": {
     "type": "git",
     "url": "https://github.com/palmetto/galaxy"
@@ -56,5 +56,19 @@
     "amqplib": "^0.10.8",
     "bullmq": "^5.70.2",
     "zod": "^4.1"
+  },
+  "peerDependenciesMeta": {
+    "@google-cloud/pubsub": {
+      "optional": true
+    },
+    "amqp-connection-manager": {
+      "optional": true
+    },
+    "amqplib": {
+      "optional": true
+    },
+    "bullmq": {
+      "optional": true
+    }
   }
 }