@palmetto/pubsub 2.1.3 → 2.2.1

package/README.md

@@ -61,6 +61,10 @@ yarn add @palmetto/pubsub zod
    };
 
    await publisher.publish(schemaConfig, message);
+
+   // publish a batch of messages quicker than sending the one at a time:
+   const messages: Model[] = [...];
+   await publisher.publish(schemaConfig, messages);
    ```
 
 1. Create subscriber and subscriber to a message
@@ -108,6 +112,15 @@ yarn add @palmetto/pubsub zod
    await subscriber.startSubscribe(schemaConfig, onMessage);
    ```
 
+1. Register with dd-trace for improved trace logs
+
+   ```ts
+   import tracer from "dd-trace";
+   import { registerDdTrace } from "@palmetto/pubsub";
+
+   registerDdTrace(tracer);
+   ```
+
 ### Message logging
 
 For each message you can indicate how to log messages when they are published or handled by a subscriber. Use the `messageLogLevel` property of the queue configuration object.

package/dist/bullmq/publisher.d.ts

@@ -8,7 +8,7 @@ export declare class BullMqPublisher implements PublisherProvider {
     constructor(connection: bullmq.ConnectionOptions, logger: Logger);
     readonly transport: string;
     private getQueue;
-    publish(config: BullMqQueueConfiguration, message: string): Promise<void>;
+    publish(config: BullMqQueueConfiguration, messages: string[]): Promise<void>;
     init(config: BullMqQueueConfiguration): Promise<void>;
     close(): Promise<void>;
     enrichPublishedMesssageLog(config: BullMqQueueConfiguration): Record<string, unknown>;

package/dist/bullmq/publisher.js

@@ -36,18 +36,20 @@ class BullMqPublisher {
             return queue;
         });
     }
-    publish(config, message) {
+    publish(config, messages) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
             const queue = yield this.getQueue(config);
-            (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `Publishing message to ${queue.name} - ${message}`);
-            yield queue.add(config.job || connection_js_1.BULLMQ_DEFAULTJOB, message, {
-                attempts: config.retries,
-                backoff: {
-                    type: "fixed",
-                    delay: config.retryDelay || 30000,
-                },
-            });
+            yield Promise.all(messages.map((message) => __awaiter(this, void 0, void 0, function* () {
+                var _a, _b;
+                (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `Publishing message to ${queue.name} - ${message}`);
+                yield queue.add(config.job || connection_js_1.BULLMQ_DEFAULTJOB, message, {
+                    attempts: config.retries,
+                    backoff: {
+                        type: "fixed",
+                        delay: config.retryDelay || 30000,
+                    },
+                });
+            })));
         });
     }
     init(config) {

package/dist/bullmq/pubsub.d.ts

@@ -6,7 +6,7 @@ export declare class BullMqPubSubProvider implements PubSubProvider {
     private readonly subscriber;
     constructor(connection: ConnectionOptions, logger: Logger);
     readonly transport: string;
-    publish(config: BullMqQueueConfiguration, message: string): Promise<void>;
+    publish(config: BullMqQueueConfiguration, messages: string[]): Promise<void>;
     startSubscribe(config: BullMqQueueConfiguration, onMessage: (s: string, context: MessageContext) => Promise<MessageResult> | MessageResult): Promise<StopSubscribe>;
     close(): Promise<void>;
     init(config: BullMqQueueConfiguration): Promise<void>;

package/dist/bullmq/pubsub.js

@@ -19,8 +19,8 @@ class BullMqPubSubProvider {
         this.publisher = new publisher_js_1.BullMqPublisher(connection, logger);
         this.subscriber = new subscriber_js_1.BullMqSubscriber(connection, logger);
     }
-    publish(config, message) {
-        return this.publisher.publish(config, message);
+    publish(config, messages) {
+        return this.publisher.publish(config, messages);
     }
     startSubscribe(config, onMessage) {
         return this.subscriber.startSubscribe(config, onMessage);

package/dist/dd-trace.wrapper.d.ts

@@ -0,0 +1,61 @@
+interface Span {
+    /**
+     * Sets the end timestamp and finalizes Span state.
+     *
+     * With the exception of calls to Span.context() (which are always allowed),
+     * finish() must be the last call made to any span instance, and to do
+     * otherwise leads to undefined behavior.
+     */
+    finish(): void;
+    /**
+     * Adds a single tag to the span.  See `addTags()` for details.
+     *
+     * @param {string} key
+     * @param {any} value
+     */
+    setTag(key: string, value: unknown): this;
+}
+interface SpanOptions {
+    /**
+     * a parent SpanContext (or Span, for convenience) that the newly-started
+     * span will be the child of (per REFERENCE_CHILD_OF). If specified,
+     * `references` must be unspecified.
+     */
+    childOf: Span | null;
+    /**
+     * set of key-value pairs which will be set as tags on the newly created
+     * Span. Ownership of the object is passed to the created span for
+     * efficiency reasons (the caller should not modify this object after
+     * calling startSpan).
+     */
+    tags: {
+        [key: string]: unknown;
+    };
+}
+/**
+ * The Datadog Scope Manager. This is used for context propagation.
+ */
+interface Scope {
+    /**
+     * Get the current active span or null if there is none.
+     *
+     * @returns {Span} The active span.
+     */
+    active(): Span | null;
+}
+interface Tracer {
+    /**
+     * Starts and returns a new Span representing a logical unit of work.
+     * @param {string} name The name of the operation.
+     * @param {SpanOptions} [options] Options for the newly created span.
+     * @returns {Span} A new Span object.
+     */
+    startSpan(name: string, options: SpanOptions): Span;
+    /**
+     * Returns a reference to the current scope.
+     */
+    scope(): Scope;
+}
+export declare function getTracer(): Tracer;
+export declare function registerDdTrace(tracer: Tracer): void;
+export {};

package/dist/dd-trace.wrapper.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getTracer = getTracer;
+exports.registerDdTrace = registerDdTrace;
+class NoOpSpan {
+    finish() {
+        // no-op
+    }
+    setTag(_key, _value) {
+        return this;
+    }
+}
+class NoOpScope {
+    active() {
+        return null;
+    }
+}
+class NoOpTracer {
+    constructor() {
+        this.noOpSpan = new NoOpSpan();
+        this.noOpScope = new NoOpScope();
+    }
+    startSpan(_name, _options) {
+        return this.noOpSpan;
+    }
+    scope() {
+        return this.noOpScope;
+    }
+}
+let tracerInstance = new NoOpTracer();
+function getTracer() {
+    return tracerInstance;
+}
+function registerDdTrace(tracer) {
+    tracerInstance = tracer;
+}

package/dist/errors.d.ts

@@ -10,6 +10,8 @@ export declare class MissingPubSubProviderError extends Error {
     static isMissingPubSubProviderError(err: unknown): err is MissingPubSubProviderError;
 }
 export declare class PublishError extends Error {
+    readonly failedMessages: string[];
+    constructor(message: string, failedMessages: string[]);
     static isPublishError(err: unknown): err is PublishError;
 }
 export declare class SchemaValidationError extends Error {

package/dist/errors.js

@@ -24,6 +24,10 @@ class MissingPubSubProviderError extends Error {
 }
 exports.MissingPubSubProviderError = MissingPubSubProviderError;
 class PublishError extends Error {
+    constructor(message, failedMessages) {
+        super(message);
+        this.failedMessages = failedMessages;
+    }
     static isPublishError(err) {
         return err instanceof PublishError;
     }

package/dist/interfaces.d.ts

@@ -23,7 +23,7 @@ export type BaseMessage = z.infer<typeof IdMetaSchema>;
 export interface PublisherProvider {
     transport: string;
     init(config: PubSubConfiguration): Promise<void> | void;
-    publish(config: PubSubConfiguration, message: string): Promise<void> | void;
+    publish(config: PubSubConfiguration, messages: string[]): Promise<void> | void;
     close(): Promise<void> | void;
     enrichPublishedMesssageLog(config: PubSubConfiguration): Record<string, unknown>;
 }

package/dist/main.d.ts

@@ -1,3 +1,4 @@
+export * from "./dd-trace.wrapper.js";
 export * from "./errors.js";
 export * from "./interfaces.js";
 export * from "./publisher.js";

package/dist/main.js

@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./dd-trace.wrapper.js"), exports);
 __exportStar(require("./errors.js"), exports);
 __exportStar(require("./interfaces.js"), exports);
 __exportStar(require("./publisher.js"), exports);

package/dist/publisher.d.ts

@@ -7,7 +7,7 @@ export declare class Publisher {
     addProvider(provider: PublisherProvider): void;
     removeProvider(providerOrTransport: PublisherProvider | string): boolean;
     init(config: PubSubConfiguration): Promise<void>;
-    publish(config: PubSubConfiguration, message: BaseMessage): Promise<void>;
+    publish(config: PubSubConfiguration, message: BaseMessage | BaseMessage[]): Promise<void>;
     close(): Promise<void>;
     private getProvider;
     private assertSchema;

package/dist/publisher.js

@@ -15,6 +15,7 @@ const uuid_1 = require("uuid");
 const crypto_1 = require("crypto");
 const errors_js_1 = require("./errors.js");
 const message_logger_js_1 = require("./message-logger.js");
+const dd_trace_wrapper_js_1 = require("./dd-trace.wrapper.js");
 class Publisher {
     constructor(logger, providers) {
         var _a, _b;
@@ -53,43 +54,71 @@ class Publisher {
     }
     publish(config, message) {
         return __awaiter(this, void 0, void 0, function* () {
-            const provider = this.getProvider(config);
-            if (!message.id) {
-                message.id = (0, uuid_1.v4)();
+            const tracer = (0, dd_trace_wrapper_js_1.getTracer)();
+            const span = tracer.startSpan("pubsub.publish", {
+                childOf: tracer.scope().active(),
+                tags: {
+                    "pubsub.transport": config.transport,
+                    "pubsub.name": config.name,
+                },
+            });
+            try {
+                const provider = this.getProvider(config);
+                const { schema, schemaId } = this.assertSchema(config);
+                if (!Array.isArray(message)) {
+                    message = [message];
+                }
+                const jsons = message.map((msg) => {
+                    if (!msg.id) {
+                        msg.id = (0, uuid_1.v4)();
+                    }
+                    if (!msg.meta) {
+                        msg.meta = {
+                            createdAt: new Date().toISOString(),
+                            publishedBy: "",
+                            schemaId: schemaId,
+                        };
+                    }
+                    else {
+                        msg.meta.schemaId = schemaId;
+                    }
+                    const check = schema.safeEncode(msg);
+                    if (!check.success) {
+                        (0, message_logger_js_1.logMessage)({
+                            note: "Publish message failed schema validation",
+                            message: message,
+                            level: "error",
+                            logger: this.logger,
+                            extra: Object.assign({ transport: provider.transport, name: config.name }, provider.enrichPublishedMesssageLog(config)),
+                        });
+                        throw new errors_js_1.SchemaValidationError(`Schema did not accept the published message: ${check.error.message}`);
+                    }
+                    const json = JSON.stringify(check.data);
+                    return { json, data: check.data };
+                });
+                const start = (0, message_logger_js_1.startTiming)();
+                yield provider.publish(config, jsons.map((j) => j.json));
+                const duration = (0, message_logger_js_1.getDuration)(start);
+                jsons.forEach((msg) => {
+                    (0, message_logger_js_1.logMessage)({
+                        note: "Published message",
+                        message: msg.data,
+                        level: config.messageLogLevel,
+                        logger: this.logger,
+                        extra: Object.assign({ transport: provider.transport, name: config.name, durationMs: duration }, provider.enrichPublishedMesssageLog(config)),
+                    });
+                });
+                span.setTag("pubsub.duration_ms", duration);
+                span.setTag("pubsub.message_count", message.length);
+                span.setTag("pubsub.success", true);
             }
-            const { schema, schemaId } = this.assertSchema(config);
-            if (!message.meta) {
-                message.meta = {
-                    createdAt: new Date().toISOString(),
-                    publishedBy: "",
-                    schemaId: schemaId,
-                };
+            catch (err) {
+                span.setTag("pubsub.success", false);
+                throw err;
             }
-            else {
-                message.meta.schemaId = schemaId;
+            finally {
+                span.finish();
             }
-            const check = schema.safeEncode(message);
-            if (!check.success) {
-                (0, message_logger_js_1.logMessage)({
-                    note: "Publish message failed schema validation",
-                    message: message,
-                    level: "error",
-                    logger: this.logger,
-                    extra: Object.assign({ transport: provider.transport, name: config.name }, provider.enrichPublishedMesssageLog(config)),
-                });
-                throw new errors_js_1.SchemaValidationError(`Schema did not accept the published message: ${check.error.message}`);
-            }
-            const json = JSON.stringify(check.data);
-            const start = (0, message_logger_js_1.startTiming)();
-            yield provider.publish(config, json);
-            const duration = (0, message_logger_js_1.getDuration)(start);
-            (0, message_logger_js_1.logMessage)({
-                note: "Published message",
-                message: check.data,
-                level: config.messageLogLevel,
-                logger: this.logger,
-                extra: Object.assign({ transport: provider.transport, name: config.name, durationMs: duration }, provider.enrichPublishedMesssageLog(config)),
-            });
         });
     }
     close() {
@@ -108,10 +137,12 @@ class Publisher {
         return provider;
     }
     assertSchema(config) {
+        var _a, _b;
         const { schema } = config;
         let hash = this.hashes.get(schema);
         if (!hash) {
-            const jsonSchema = JSON.stringify(zod_1.z.toJSONSchema(schema, { io: "input" }), null, 3);
+            (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `Calculating schema id for ${config.name}`);
+            const jsonSchema = JSON.stringify(zod_1.z.toJSONSchema(schema, { io: "input" }), null, 2);
             hash = (0, crypto_1.createHash)("sha256").update(jsonSchema).digest("hex");
             this.hashes.set(schema, hash);
         }

package/dist/rabbitmq/publisher.d.ts

@@ -24,9 +24,10 @@ export declare class RabbitMqPublisher implements PublisherProvider {
      * @param message The JSON message to send
      * @returns A promise that is completed when the message is published
      */
-    publish(config: RabbitQueueExchangeConfiguration, message: string): Promise<void>;
+    publish(config: RabbitQueueExchangeConfiguration, messages: string[]): Promise<void>;
     enrichPublishedMesssageLog(config: RabbitQueueExchangeConfiguration): Record<string, unknown>;
     private publishToExchange;
     private publishToQueue;
+    private sendMessages;
     close(): Promise<void> | undefined;
 }

package/dist/rabbitmq/publisher.js

@@ -78,14 +78,14 @@ class RabbitMqPublisher {
      * @param message The JSON message to send
      * @returns A promise that is completed when the message is published
      */
-    publish(config, message) {
+    publish(config, messages) {
         return __awaiter(this, void 0, void 0, function* () {
             const channel = yield this.getChannel(config);
             if (config.publishToSpecificQueue) {
-                yield this.publishToQueue(channel, config.publishToSpecificQueue, message);
+                yield this.publishToQueue(channel, config.publishToSpecificQueue, messages);
             }
             else {
-                yield this.publishToExchange(channel, config, message);
+                yield this.publishToExchange(channel, config, messages);
             }
         });
     }
@@ -100,28 +100,31 @@ class RabbitMqPublisher {
             routingKey: (0, config_js_1.getRoutingKey)(config),
         };
     }
-    publishToExchange(channel, config, message) {
+    publishToExchange(channel, config, messages) {
         return __awaiter(this, void 0, void 0, function* () {
             const exchangeName = (0, config_js_1.getExchangeName)(config);
-            const ok = yield channel.publish(exchangeName, (0, config_js_1.getRoutingKey)(config), Buffer.from(message, "utf8"), {
-                contentType: "application/json",
-                timestamp: new Date().valueOf(),
-                persistent: true,
-            });
-            if (!ok) {
-                throw new errors_js_1.PublishError(`RabbitMq publish to ${exchangeName} failed`);
-            }
+            const routingKey = (0, config_js_1.getRoutingKey)(config);
+            return this.sendMessages(messages, (buf, options) => channel.publish(exchangeName, routingKey, buf, options), "publish to exchange", exchangeName);
         });
     }
-    publishToQueue(channel, queue, message) {
+    publishToQueue(channel, queue, messages) {
+        return this.sendMessages(messages, (buf, options) => channel.sendToQueue(queue, buf, options), "send to queue", queue);
+    }
+    sendMessages(messages, sender, action, name) {
         return __awaiter(this, void 0, void 0, function* () {
-            const ok = yield channel.sendToQueue(queue, Buffer.from(message, "utf8"), {
+            const options = {
                 contentType: "application/json",
                 timestamp: new Date().valueOf(),
                 persistent: true,
-            });
-            if (!ok) {
-                throw new errors_js_1.PublishError(`RabbitMq publish to queue:${queue} failed`);
+            };
+            const oks = yield Promise.all(messages.map((message) => sender(Buffer.from(message, "utf8"), options)));
+            const failedMessages = oks
+                .map((ok, index) => ({ ok, index }))
+                .filter(({ ok }) => !ok)
+                .map(({ index }) => messages[index]);
+            if (failedMessages.length > 0) {
+                const allOrSome = failedMessages.length === messages.length ? "all" : "some";
+                throw new errors_js_1.PublishError(`RabbitMq ${action} ${name} failed for ${allOrSome} messages`, failedMessages);
             }
         });
     }

package/dist/rabbitmq/pubsub.d.ts

@@ -9,7 +9,7 @@ export declare class RabbitMqPubSubProvider implements PubSubProvider {
     private readonly subscriber;
     constructor(connection: RabbitMqConnection, logger: Logger);
     readonly transport: string;
-    publish(config: RabbitQueueExchangeConfiguration, message: string): Promise<void>;
+    publish(config: RabbitQueueExchangeConfiguration, messages: string[]): Promise<void>;
     startSubscribe(config: RabbitQueueExchangeConfiguration, onMessage: (s: string, context: MessageContext) => Promise<MessageResult> | MessageResult): StopSubscribe;
     close(): Promise<void>;
     init(config: RabbitQueueExchangeConfiguration): Promise<void>;

package/dist/rabbitmq/pubsub.js

@@ -22,8 +22,8 @@ class RabbitMqPubSubProvider {
         this.publisher = new publisher_js_1.RabbitMqPublisher(connection, logger);
         this.subscriber = new subscriber_js_1.RabbitMqSubscriber(connection, logger);
     }
-    publish(config, message) {
-        return this.publisher.publish(config, message);
+    publish(config, messages) {
+        return this.publisher.publish(config, messages);
     }
     startSubscribe(config, onMessage) {
         return this.subscriber.startSubscribe(config, onMessage);

package/dist/subscriber.js

@@ -15,6 +15,7 @@ const errors_js_1 = require("./errors.js");
 const events_1 = require("events");
 const message_logger_js_1 = require("./message-logger.js");
 const create_log_error_payload_js_1 = require("./create-log-error-payload.js");
+const dd_trace_wrapper_js_1 = require("./dd-trace.wrapper.js");
 class SubscribedMessage {
     constructor(stop) {
         this.stop = stop;
@@ -56,6 +57,14 @@ class Subscriber {
                 throw new errors_js_1.MissingPubSubProviderError(`No provider configured for ${config.transport}`);
             }
             const transform = (jsonStr, context) => __awaiter(this, void 0, void 0, function* () {
+                const tracer = (0, dd_trace_wrapper_js_1.getTracer)();
+                const span = tracer.startSpan("pubsub.handleMessage", {
+                    childOf: tracer.scope().active(),
+                    tags: {
+                        "pubsub.transport": config.transport,
+                        "pubsub.name": config.name,
+                    },
+                });
                 const start = (0, message_logger_js_1.startTiming)();
                 const jsonObject = JSON.parse(jsonStr);
                 const r = schema.safeDecode(jsonObject);
@@ -77,6 +86,8 @@ class Subscriber {
                     };
                     this.events.emit("schemaError", handledEventContext);
                     this.events.emit("messageHandled", handledEventContext);
+                    span.setTag("pubsub.success", false);
+                    span.finish();
                     return interfaces_js_1.MessageResult.Fail;
                 }
                 try {
@@ -98,6 +109,9 @@ class Subscriber {
                             result }, provider.enrichHandledMesssageLog(config)),
                     });
                     this.events.emit("messageHandled", eventContext);
+                    span.setTag("pubsub.duration_ms", durationMs);
+                    span.setTag("pubsub.success", true);
+                    span.finish();
                     return result;
                 }
                 catch (err) {
@@ -117,6 +131,8 @@ class Subscriber {
                         err,
                     };
                     this.events.emit("messageHandled", eventContext);
+                    span.setTag("pubsub.success", false);
+                    span.finish();
                     throw err;
                 }
             });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@palmetto/pubsub",
-  "version": "2.1.3",
+  "version": "2.2.1",
   "main": "./dist/main.js",
   "scripts": {
     "lint": "yarn run -T eslint --fix ./src",