@palmetto/pubsub 3.2.0 → 3.2.1

package/dist/bullmq/subscriber.js

@@ -75,7 +75,7 @@ class BullMqSubscriber {
                         attemptsMade: job.attemptsMade,
                         firstSent: new Date(job.timestamp),
                         lastSent: undefined,
-                        willRetryOnFailure: this.willRetryJobOnFailure(job),
+                        willAttemptRetry: this.willRetryJobOnFailure(job),
                     };
                     const result = yield onMessage(job.data, context);
                     if (result === interfaces_js_1.MessageResult.Ok) {

package/dist/gcppubsub/config.d.ts

@@ -1,4 +1,4 @@
-import { type PubSub } from "@google-cloud/pubsub";
+import { type CreateSubscriptionOptions, type SubscriptionOptions, type PubSub, Topic } from "@google-cloud/pubsub";
 import { MessageContext, PubSubConfiguration } from "../interfaces.js";
 import { GCP_PUBSUB_TRANSPORT } from "./connection.js";
 export interface GcpPubSubConfiguration extends PubSubConfiguration {
@@ -10,6 +10,10 @@ export interface GcpPubSubConfiguration extends PubSubConfiguration {
      * The Pub/Sub topic name. Defaults to `name` if omitted.
      */
     topicName?: string;
+    /**
+     * The Pub/Sub dead-letter topic name. Defaults to `${name}.dl` if omitted.
+     */
+    deadLetterTopicName?: string;
     /**
      * The Pub/Sub subscription name. Required for subscribers. Defaults to `name` if omitted.
      */
@@ -18,7 +22,31 @@ export interface GcpPubSubConfiguration extends PubSubConfiguration {
      * Flow control: max outstanding messages (default: 5)
      */
     maxMessages?: number;
+    /**
+     * Maximum number of times to retry a message.
+     * Default: 0, no retries
+     * Max: 99 retries, see max_delivery_attempts in GCP docs
+     * Infinite retries: -1
+     */
+    retries?: number;
+    /**
+     * Maximum amount of time to wait, in milliseconds, before retrying a message. Default: 600_000 (10 minutes)
+     */
+    maxRetryDelay?: number;
+    /**
+     * Subscription settings to override defaults when creating a new subscription
+     */
+    createSubscriptionOptions?: CreateSubscriptionOptions;
+    /**
+     * subscription options to override defaults when starting a new subscription
+     */
+    subscriptionOptions?: SubscriptionOptions;
+    /**
+     * During shutdown, pending acks will be waited for up to this amount of time (in milliseconds) after the last message is received before the subscription is closed. Default: 1_000 (1 second)
+     */
+    shutdownDelay?: number;
 }
+export declare const RETRY_FOREVER = -1;
 export type TopicType = "default" | "dead-letter";
 export declare const TopicNameExtensions: Record<TopicType, string | undefined>;
 export interface GcpPubSubMessageContext extends MessageContext {
@@ -31,4 +59,8 @@ export declare function getTopicName(config: GcpPubSubConfiguration, topicType:
 export declare function getSubscriptionName(config: GcpPubSubConfiguration): string;
 export declare function isGcpPubSubConfiguration(config: PubSubConfiguration): config is GcpPubSubConfiguration;
 export declare function ensureTopic(client: PubSub, topicName: string): Promise<void>;
-export declare function ensureSubscription(client: PubSub, topicName: string, subscriptionName: string, retryDelay: number): Promise<void>;
+export declare function ensureSubscription(client: PubSub, config: GcpPubSubConfiguration): Promise<{
+    deadLetterTopic?: Topic;
+    retries: number;
+    created: boolean;
+}>;

package/dist/gcppubsub/config.js

@@ -9,19 +9,24 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TopicNameExtensions = void 0;
+exports.TopicNameExtensions = exports.RETRY_FOREVER = void 0;
 exports.getTopicName = getTopicName;
 exports.getSubscriptionName = getSubscriptionName;
 exports.isGcpPubSubConfiguration = isGcpPubSubConfiguration;
 exports.ensureTopic = ensureTopic;
 exports.ensureSubscription = ensureSubscription;
+const pubsub_1 = require("@google-cloud/pubsub");
 const connection_js_1 = require("./connection.js");
+exports.RETRY_FOREVER = -1;
 exports.TopicNameExtensions = {
     "dead-letter": ".dl",
     default: undefined,
 };
 function getTopicName(config, topicType) {
     var _a, _b;
+    if (topicType === "dead-letter" && config.deadLetterTopicName) {
+        return config.deadLetterTopicName;
+    }
     const baseName = (_a = config.topicName) !== null && _a !== void 0 ? _a : config.name;
     const extension = (_b = exports.TopicNameExtensions[topicType]) !== null && _b !== void 0 ? _b : "";
     return `${baseName}${extension}`;
@@ -41,19 +46,81 @@ function ensureTopic(client, topicName) {
         }
     });
 }
-function ensureSubscription(client, topicName, subscriptionName, retryDelay) {
+function ensureSubscription(client, config) {
     return __awaiter(this, void 0, void 0, function* () {
+        var _a, _b, _c, _d, _e;
+        const topic = client.topic(getTopicName(config, "default"));
+        const subscriptions = (yield topic.getSubscriptions({
+            pageSize: 1000,
+        }))[0];
+        const subscriptionName = getSubscriptionName(config);
+        const subscription = subscriptions.find((s) => s.name.endsWith(`/${subscriptionName}`));
+        if (subscription) {
+            const meta = (_a = subscription.metadata) !== null && _a !== void 0 ? _a : (yield subscription.getMetadata())[0];
+            let deadLetterTopic = undefined;
+            let retries;
+            if ((_b = meta.deadLetterPolicy) === null || _b === void 0 ? void 0 : _b.deadLetterTopic) {
+                const dlp = pubsub_1.protos.google.pubsub.v1.DeadLetterPolicy.create(meta.deadLetterPolicy);
+                if (config.retries !== undefined && config.retries < 0) {
+                    // If retries is negative, we want to retry forever, but GCP Pub/Sub doesn't support that, so we set it to the max allowed by the dead-letter policy
+                    retries = dlp.maxDeliveryAttempts - 1;
+                }
+                else {
+                    // config.retries can override maxDeliveryAttempts, but it can't exceed the max allowed by the dead-letter policy
+                    retries = Math.min((_c = config.retries) !== null && _c !== void 0 ? _c : 0, dlp.maxDeliveryAttempts - 1);
+                }
+                deadLetterTopic = client.topic(dlp.deadLetterTopic);
+            }
+            else {
+                retries = exports.RETRY_FOREVER;
+            }
+            return {
+                deadLetterTopic,
+                retries,
+                created: false,
+            };
+        }
+        const topicName = getTopicName(config, "default");
+        const dlTopicName = getTopicName(config, "dead-letter");
+        const minRetryDelay = (_d = config.retryDelay) !== null && _d !== void 0 ? _d : 30000;
+        const maxRetryDelay = config.maxRetryDelay;
+        const maxDeliveryAttempts = ((_e = config.retries) !== null && _e !== void 0 ? _e : 0) + 1;
+        const enableDeadLetter = maxDeliveryAttempts > 0;
+        if (enableDeadLetter) {
+            yield ensureSubscription(client, Object.assign(Object.assign({}, config), { retries: exports.RETRY_FOREVER, name: dlTopicName, topicName: dlTopicName, deadLetterTopicName: undefined, subscriptionName: `${dlTopicName}.sub`, retryDelay: minRetryDelay, maxRetryDelay }));
+        }
         yield ensureTopic(client, topicName);
-        const subscriptions = (yield client.getSubscriptions())[0];
-        if (!subscriptions.some((s) => s.name.endsWith(`/${subscriptionName}`))) {
-            yield client.createSubscription(topicName, subscriptionName, {
-                retryPolicy: {
-                    minimumBackoff: {
-                        nanos: (retryDelay % 1000) * 1000000,
-                        seconds: Math.floor(retryDelay / 1000),
-                    },
+        const enableRetryPolicy = minRetryDelay > 0 || maxRetryDelay !== undefined;
+        const deadLetterTopic = enableDeadLetter
+            ? client.topic(dlTopicName)
+            : undefined;
+        const options = Object.assign(Object.assign(Object.assign({}, (enableDeadLetter
+            ? {
+                deadLetterPolicy: {
+                    deadLetterTopic: deadLetterTopic === null || deadLetterTopic === void 0 ? void 0 : deadLetterTopic.name,
+                    maxDeliveryAttempts: Math.max(maxDeliveryAttempts, 5),
                 },
-            });
-        }
+            }
+            : {})), (enableRetryPolicy
+            ? {
+                retryPolicy: Object.assign({ minimumBackoff: {
+                        nanos: (minRetryDelay % 1000) * 1000000,
+                        seconds: Math.floor(minRetryDelay / 1000),
+                    } }, (maxRetryDelay !== undefined
+                    ? {
+                        maximumBackoff: {
+                            nanos: (maxRetryDelay % 1000) * 1000000,
+                            seconds: Math.floor(maxRetryDelay / 1000),
+                        },
+                    }
+                    : {})),
+            }
+            : {})), config.createSubscriptionOptions);
+        yield topic.createSubscription(subscriptionName, options);
+        return {
+            deadLetterTopic,
+            retries: maxDeliveryAttempts > 0 ? maxDeliveryAttempts - 1 : exports.RETRY_FOREVER,
+            created: true,
+        };
     });
 }

package/dist/gcppubsub/subscriber.js

@@ -10,14 +10,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GcpPubSubSubscriber = void 0;
+const pubsub_1 = require("@google-cloud/pubsub");
 const trace_1 = require("@palmetto/trace");
 const interfaces_js_1 = require("../interfaces.js");
 const errors_js_1 = require("../errors.js");
 const create_log_error_payload_js_1 = require("../create-log-error-payload.js");
 const config_js_1 = require("./config.js");
 const connection_js_1 = require("./connection.js");
-const RETRY_COUNT_ATTR = "x-retry-count";
-const FIRST_PUBLISHED_ATTR = "x-first-published";
+const subscriber_js_1 = require("@google-cloud/pubsub/build/src/subscriber.js");
 const SHUTDOWN_DELAY = 1000;
 class GcpPubSubSubscriber {
     constructor(connection, logger) {
@@ -29,22 +29,33 @@ class GcpPubSubSubscriber {
     }
     startSubscribe(config, onMessage) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
+            var _a, _b, _c, _d;
             const subscriptionName = (0, config_js_1.getSubscriptionName)(config);
             if (this.subscriptions.has(subscriptionName)) {
                 throw new errors_js_1.AlreadySubscribingError(subscriptionName);
             }
             const topicName = (0, config_js_1.getTopicName)(config, "default");
-            const dlqTopicName = (0, config_js_1.getTopicName)(config, "dead-letter");
-            const retryDelay = (_a = config.retryDelay) !== null && _a !== void 0 ? _a : 30000;
-            yield (0, config_js_1.ensureSubscription)(this.connection.client, topicName, subscriptionName, retryDelay);
-            yield (0, config_js_1.ensureTopic)(this.connection.client, dlqTopicName);
-            const subscription = this.connection.client.subscription(subscriptionName, {
-                flowControl: {
-                    maxMessages: (_b = config.maxMessages) !== null && _b !== void 0 ? _b : 5,
-                },
-            });
-            const dlTopic = this.connection.client.topic(dlqTopicName);
+            const { deadLetterTopic, retries } = yield (0, config_js_1.ensureSubscription)(this.connection.client, config);
+            if (retries !== ((_a = config.retries) !== null && _a !== void 0 ? _a : 0)) {
+                const configRetries = (_b = config.retries) !== null && _b !== void 0 ? _b : 0;
+                const log = {
+                    message: `GCP Pub/Sub subscription ${subscriptionName} retry configuration mismatch.`,
+                    extra: {
+                        subscriptionName,
+                        configRetries,
+                        subscriptionRetries: retries,
+                    },
+                };
+                this.logger.warn(log);
+            }
+            const subscription = this.connection.client.subscription(subscriptionName, Object.assign({ flowControl: {
+                    maxMessages: (_c = config.maxMessages) !== null && _c !== void 0 ? _c : 5,
+                }, closeOptions: {
+                    behavior: subscriber_js_1.SubscriberCloseBehaviors.WaitForProcessing,
+                    timeout: pubsub_1.Duration.from({
+                        milliseconds: (_d = config.shutdownDelay) !== null && _d !== void 0 ? _d : SHUTDOWN_DELAY,
+                    }),
+                } }, config.subscriptionOptions));
             const messageHandler = (message) => {
                 void this.consumeMessage({
                     config,
@@ -52,8 +63,8 @@ class GcpPubSubSubscriber {
                     onMessage,
                     subscriptionName,
                     topicName,
-                    dlTopic,
-                    retryDelay,
+                    deadLetterTopic,
+                    retries,
                 });
             };
             const errorHandler = (err) => {
@@ -65,12 +76,12 @@ class GcpPubSubSubscriber {
             };
             subscription.on("message", messageHandler);
             subscription.on("error", errorHandler);
-            this.subscriptions.set(subscriptionName, subscription);
+            this.subscriptions.set(subscriptionName, { subscription, config });
             this.logger.log(`GCP Pub/Sub consumer started for ${subscriptionName}`);
             return () => __awaiter(this, void 0, void 0, function* () {
                 subscription.removeListener("message", messageHandler);
                 subscription.removeListener("error", errorHandler);
-                yield this.waitForMessagesToComplete();
+                yield this.waitForMessagesToComplete(config.shutdownDelay);
                 yield subscription.close();
                 this.subscriptions.delete(subscriptionName);
                 this.logger.log(`GCP Pub/Sub consumer stopped for ${subscriptionName}`);
@@ -80,8 +91,8 @@ class GcpPubSubSubscriber {
     close() {
         return __awaiter(this, void 0, void 0, function* () {
             yield Promise.all([...this.subscriptions.entries()].map((_a) => __awaiter(this, [_a], void 0, function* ([subscriptionName, sub]) {
-                yield this.waitForMessagesToComplete();
-                yield sub.close();
+                yield this.waitForMessagesToComplete(sub.config.shutdownDelay);
+                yield sub.subscription.close();
                 this.logger.log(`GCP Pub/Sub consumer stopped for ${subscriptionName}`);
             })));
             this.subscriptions.clear();
@@ -94,27 +105,28 @@ class GcpPubSubSubscriber {
         };
     }
     consumeMessage(_a) {
-        return __awaiter(this, arguments, void 0, function* ({ config, message, onMessage, subscriptionName, topicName, dlTopic, }) {
+        return __awaiter(this, arguments, void 0, function* ({ config, message, onMessage, subscriptionName, topicName, deadLetterTopic, retries, }) {
             yield (0, trace_1.getTracer)().trace("pubsub.gcppubsub.consume", {
                 resource: `consume ${config.transport} ${subscriptionName}`,
             }, (span) => __awaiter(this, void 0, void 0, function* () {
-                var _a, _b, _c, _d, _e, _f, _g;
+                var _a, _b, _c, _d, _e, _f;
                 try {
                     const firstPublished = message.publishTime;
                     let retryCount = message.deliveryAttempt - 1;
-                    let willRetryOnFailure;
-                    if (retryCount < 0) {
+                    let willAttemptRetry;
+                    if (retryCount < 0 || retries === config_js_1.RETRY_FOREVER) {
+                        // in this case, we will retry forever
                         retryCount = 0;
-                        willRetryOnFailure = false;
+                        willAttemptRetry = true;
                     }
                     else {
-                        willRetryOnFailure = retryCount < ((_a = config.retries) !== null && _a !== void 0 ? _a : 0);
+                        willAttemptRetry = retryCount < retries;
                     }
                     const context = {
                         attemptsMade: retryCount,
                         firstSent: firstPublished,
                         lastSent: undefined,
-                        willRetryOnFailure,
+                        willAttemptRetry,
                         messageId: message.id,
                         subscriptionName,
                         topicName,
@@ -139,27 +151,37 @@ class GcpPubSubSubscriber {
                     }
                     this.lastMessageDate = Date.now();
                     if (messageResult === interfaces_js_1.MessageResult.Ok) {
-                        (_c = (_b = this.logger).debug) === null || _c === void 0 ? void 0 : _c.call(_b, logPayload);
+                        (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, logPayload);
                         message.ack();
                         return;
                     }
-                    if (messageResult === interfaces_js_1.MessageResult.Retry && willRetryOnFailure) {
+                    if (messageResult === interfaces_js_1.MessageResult.Retry && willAttemptRetry) {
                         message.nack();
-                        logPayload.message += " Retrying message via Pubsub.";
-                        (_e = (_d = this.logger).debug) === null || _e === void 0 ? void 0 : _e.call(_d, logPayload);
+                        logPayload.message += " Retrying message via gcp-pubsub.";
+                        (_d = (_c = this.logger).debug) === null || _d === void 0 ? void 0 : _d.call(_c, logPayload);
                         return;
                     }
                     // Fail: either explicit Fail or Retry with no retries remaining
                     if (messageResult === interfaces_js_1.MessageResult.Retry) {
                         logPayload.message += " No more retries.";
                     }
-                    yield dlTopic.publishMessage({
-                        data: message.data,
-                        attributes: Object.assign(Object.assign({}, message.attributes), { [RETRY_COUNT_ATTR]: retryCount.toString(), [FIRST_PUBLISHED_ATTR]: firstPublished.toISOString() }),
-                    });
-                    logPayload.message += ` Published to dead-letter topic ${dlTopic.name}.`;
-                    (_g = (_f = this.logger).debug) === null || _g === void 0 ? void 0 : _g.call(_f, logPayload);
-                    message.ack();
+                    (_f = (_e = this.logger).debug) === null || _f === void 0 ? void 0 : _f.call(_e, logPayload);
+                    if (deadLetterTopic &&
+                        (messageResult === interfaces_js_1.MessageResult.Fail || retryCount < 4)) {
+                        // Send all failed messages to the dead-letter topic
+                        // GCP Pub/Sub doesn't allow maxDeliveryAttempts less than 5, so we need to manually publish to the dead-letter topic for the first 4 retries
+                        logPayload.message += " Publishing message to dead-letter topic.";
+                        yield deadLetterTopic.publishMessage({
+                            data: message.data,
+                        });
+                        message.ack(); // ack the message to remove it from the subscription
+                    }
+                    else if (messageResult === interfaces_js_1.MessageResult.Fail) {
+                        message.ack(); // ack the message to remove it from the subscription
+                    }
+                    else {
+                        message.nack(); // nack the message to let pubsub send it to the DLQ
+                    }
                 }
                 catch (err) {
                     const logPayload = {
@@ -181,8 +203,8 @@ class GcpPubSubSubscriber {
      * Wait for in-flight message handling to complete before shutting down
      */
     waitForMessagesToComplete() {
-        return __awaiter(this, void 0, void 0, function* () {
-            const waitUntil = this.lastMessageDate + SHUTDOWN_DELAY;
+        return __awaiter(this, arguments, void 0, function* (shutdownDelay = SHUTDOWN_DELAY) {
+            const waitUntil = this.lastMessageDate + shutdownDelay;
             const waitDuration = waitUntil - Date.now();
             if (waitDuration > 0) {
                 yield new Promise((resolve) => setTimeout(resolve, waitDuration));

package/dist/interfaces.d.ts

@@ -66,15 +66,17 @@ export interface PubSubConfiguration {
 }
 export declare enum MessageResult {
     /**
-     * the message was handled successfully or can otherwise be ignored
+     * The message was handled. The transport should not deliver this message again.
      */
     Ok = "ok",
     /**
-     * The message should be retried
+     * The message should be retried by the transport.
+     * Whether the message is retried immediately or after a delay depends on the transport and configuration.
+     * If the number of retries exceeds the configured limit for the transport, the message is considered failed.
      */
     Retry = "retry",
     /**
-     * The message failed and is either discarded or stored in a dead-letter queue
+     * The message failed and is either discarded or stored in a dead-letter queue, depending on the transport and configuration.
      */
     Fail = "fail"
 }
@@ -129,8 +131,23 @@ export type PubSubProvider = PublisherProvider & SubscriberProvider;
  */
 export type PubOrSubProvider = PubSubProvider | PublisherProvider | SubscriberProvider;
 export interface MessageContext {
+    /**
+     * When the message was first published.
+     */
     firstSent: Date | undefined;
+    /**
+     * When the message was last published. Not all transports provide this information.
+     */
     lastSent: Date | undefined;
+    /**
+     * The number of times the message has been attempted to be processed. This is 0 for the first attempt, 1 for the first retry, etc.
+     * Note that for some transports, this number is an approximation based on the number of times the message has been redelivered, or may always be 0.
+     */
     attemptsMade: number | undefined;
-    willRetryOnFailure: boolean;
+    /**
+     * If the onMessage callback returns MessageResult.Retry or throws an exception:
+     * When true, the message will be retried
+     * When false, the message will not be retried
+     */
+    willAttemptRetry: boolean;
 }

package/dist/interfaces.js

@@ -14,15 +14,17 @@ exports.IdMetaSchema = zod_1.z.object({
 var MessageResult;
 (function (MessageResult) {
     /**
-     * the message was handled successfully or can otherwise be ignored
+     * The message was handled. The transport should not deliver this message again.
      */
     MessageResult["Ok"] = "ok";
     /**
-     * The message should be retried
+     * The message should be retried by the transport.
+     * Whether the message is retried immediately or after a delay depends on the transport and configuration.
+     * If the number of retries exceeds the configured limit for the transport, the message is considered failed.
      */
     MessageResult["Retry"] = "retry";
     /**
-     * The message failed and is either discarded or stored in a dead-letter queue
+     * The message failed and is either discarded or stored in a dead-letter queue, depending on the transport and configuration.
      */
     MessageResult["Fail"] = "fail";
 })(MessageResult || (exports.MessageResult = MessageResult = {}));

package/dist/message-logger.d.ts

@@ -1,6 +1,6 @@
 import { Logger, MessageLogLevel } from "./interfaces";
-export declare function startTiming(): [number, number];
-export declare function getDuration(start: [number, number]): number;
+export declare function startTiming(): bigint;
+export declare function getDuration(start: bigint): number;
 export declare function logMessage({ note, message, logger, level, extra, }: {
     note: string;
     message: unknown;

package/dist/message-logger.js

@@ -4,10 +4,10 @@ exports.startTiming = startTiming;
 exports.getDuration = getDuration;
 exports.logMessage = logMessage;
 function startTiming() {
-    return process.hrtime();
+    return process.hrtime.bigint();
 }
 function getDuration(start) {
-    return process.hrtime(start)[1] / 1000000; // convert to milliseconds
+    return Number(process.hrtime.bigint() - start) / 1000000; // convert to milliseconds
 }
 function logMessage({ note, message, logger, level = "debug", extra, }) {
     var _a;

package/dist/rabbitmq/subscriber.js

@@ -195,8 +195,8 @@ class RabbitMqSubscriber {
             const json = msg.content.toString("utf8");
             const attemptsMade = RabbitMqSubscriber.getAttemptsMade(msg);
             const sentDates = RabbitMqSubscriber.getSentDates(msg);
-            const willRetryOnFailure = attemptsMade < ((_b = subscribedMessage.config.retries) !== null && _b !== void 0 ? _b : 0);
-            const context = Object.assign(Object.assign({ attemptsMade }, sentDates), { queueName, exchangeName: msg.fields.exchange, retryExchangeName, routingKey: msg.fields.routingKey, willRetryOnFailure });
+            const willAttemptRetry = attemptsMade < ((_b = subscribedMessage.config.retries) !== null && _b !== void 0 ? _b : 0);
+            const context = Object.assign(Object.assign({ attemptsMade }, sentDates), { queueName, exchangeName: msg.fields.exchange, retryExchangeName, routingKey: msg.fields.routingKey, willAttemptRetry });
             let messageResult;
             let logPayload;
             try {
@@ -220,7 +220,7 @@ class RabbitMqSubscriber {
                 return;
             }
             if (messageResult === interfaces_js_1.MessageResult.Retry) {
-                if (willRetryOnFailure) {
+                if (willAttemptRetry) {
                     logPayload.message += " Retrying message.";
                 }
                 else {

package/dist/subscriber.js

@@ -108,13 +108,13 @@ class Subscriber {
                     (0, message_logger_js_1.logMessage)({
                         note: "Subscriber error when processing message",
                         message: decodeResult.data,
-                        level: context.willRetryOnFailure ? "warn" : "error",
+                        level: context.willAttemptRetry ? "warn" : "error",
                         logger: this.logger,
                         extra: Object.assign(Object.assign({ transport: provider.transport, name: config.name, durationMs,
                             context }, enrichedConfig), { error: (0, create_log_error_payload_js_1.createLogErrorPayload)(err) }),
                     });
-                    // in the messageHandled event, we always log as retry or fail based on willRetryOnFailure
-                    const result = context.willRetryOnFailure
+                    // in the messageHandled event, we always log as retry or fail based on willAttemptRetry
+                    const result = context.willAttemptRetry
                         ? interfaces_js_1.MessageResult.Retry
                         : interfaces_js_1.MessageResult.Fail;
                     const eventContext = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@palmetto/pubsub",
-  "version": "3.2.0",
+  "version": "3.2.1",
   "repository": {
     "type": "git",
     "url": "https://github.com/palmetto/galaxy"

package/src/gcppubsub/README.md

@@ -1,6 +1,6 @@
 # @palmetto/pubsub
 
-The GoogleCloud pubsub transport provider for @palmetto/pubsub
+The GoogleCloud Pub/Sub (gcp-pubsub) transport provider for @palmetto/pubsub
 
 ## Installation
 
@@ -48,7 +48,7 @@ yarn add @palmetto/pubsub @google-cloud/pubsub zod
    ]);
    ```
 
-5. Create the queue configuration
+5. Create the topic/subscriber configuration
 
    ```ts
    import {
@@ -66,8 +66,10 @@ yarn add @palmetto/pubsub @google-cloud/pubsub zod
      subscriptionName: "my-subscription-name", // default: topicName or name
      schema: MyModelSchema,
      transport: GCP_PUBSUB_TRANSPORT,
-     retries: 10,
-     retryDelay: 1_000, // Note: retryDelay must be configured as a retry policy of the subscription. pubsub will create the subscription if it does not exist, but won't update an existing subscription.
+     retries: 10, // Note: enables dead-letter policy. Use RETRY_FOREVER to disable dead-letter. @palmetto/pubsub will not update an existing subscription.
+     retryDelay: 1_000, // Note: retryDelay is minimum retry milliseconds using the exponential backoff retry policy. @palmetto/pubsub will not update an existing subscription.
+     createSubscriptionOptions: { ... }, // set the specific createSubscription() options you want to use [optional]
+     subscriptionOptions: { ... }, // set any specific subscribe() options you want to use [optional]
    };
    ```
 
@@ -87,7 +89,7 @@ yarn add @palmetto/pubsub @google-cloud/pubsub zod
    await publisher.publish(config, message);
    ```
 
-8. Subscribe to a queue
+8. Subscribe to a topic
 
    ```ts
    import { GcpPubSubMessageContext } from "@palmetto/pubsub";
@@ -114,21 +116,101 @@ yarn add @palmetto/pubsub @google-cloud/pubsub zod
    });
    ```
 
-## Message failures
+## Message Handlers
 
-Message failures in Google Cloud pubsub subscribers are handled by pubsub's retry policies. Messages are `nack()`'ed and retried (or discarded) based on the GCP subscription's retry policy.
+Message handlers can return one of 3 values:
+
+- `MessageResult.Ok` - the message can be removed from subscription. Messages are always `ack()`'ed.
+- `MessageResult.Retry` - the message can be retried, depending on the dead-letter/retry configuration. Messages are typically `nack()`'ed to use gcp-pubsub dead-letter policy. [See below](#retrying-messages)
+- `MessageResult.Fail` - Remove it from the subscription and publish it to the dead-letter topic (if available). Messages are always `ack()`'ed. [See below](#failing-messages)
+
+If a message handler throws an exception, this is the same as if it returned `MessageResult.Retry`.
 
 ### Retrying messages
 
-These properties configure message retries when `@palmetto/pubsub` creates the subscription. If `@palmetto/pubsub` didn't create the subscription, whatever retry policy in place is what will be used (by default there are no retries).
+By default gcp-pubsub subscriptions retry messages immediately, but `@palmetto/pubsub` subscriptions default to a 30 second retry policy.
+
+- minimum backoff: default 30 seconds
+- maximum backoff: default 10 minutes (gcp-pubsub default)
+
+You can adjust these defaults using these two configuration settings:
+
+- `retryDelay`
+  - 0 : immediate retry
+  - 1 to 600_000 : enable exponential backoff retry policy (values over 600_000 [10 minutes] are not supported by gcp-pubsub)
+- `maxRetryDelay`
+  - any value here enables exponential backoff retry policy
+  - be sure this value is equal to or larger than the `retryDelay`
+
+Messages can be retried when the handler throws an exception or when it returns `MessageResult.Retry`.
+
+The `retries` property configures the subscription retry policy when `@palmetto/pubsub` creates the subscription. If `@palmetto/pubsub` didn't create the subscription, whatever retry policy in place is what will be used (by default there are infinite retries).
+
+NOTE: If the subscription already exists, and there is no dead-letter policy, the configured `retries` value is ignored and assumes `RETRY_FOREVER` instead.
+
+The following special values define how `@palmetto/pubsub` creates the subscription retry policy:
+
+#### `{ retries: RETRY_FOREVER }`
+
+Use `RETRY_FOREVER` or `-1` to disable dead-letter policy.
+
+When a message handler throws an exception or returns `MessageResult.Retry` the message is retried. Messages are retried until the message retention period expires.
+
+When a message handler returns `MessageResult.Fail` or `MessageResult.Ok`, `ack()` is called on the message to remove it from the subscription.
+
+The `messageContext` includes `{ attemptsMade: 0 }` on every message.
+
+#### `{ retries: undefined }` or `{ retries: 0 }`
+
+There are no retries, but dead-letter topic is enabled.
+
+An exception or return values of `MessageResult.Fail` or `MessageResult.Retry` the message is published to the dead-letter topic and `ack()` is called on the message to remove it from the subscription.
+
+The `messageContext` includes `{ attemptsMade: 0 }` on the first message and will be a number greater than 0 if the message is retried due to the race condition.
 
-- `retries` : specifies the number of retries after which the message is sent to the dead-letter topic. When this value is undefined, the message will not retry.
-- `retryDelay`: specifies the number of milliseconds between retries.
+#### `{ retries: 1, 2 or 3 }`
 
-Failed messages are `nack()`'ed to pubsub and they are retried with the delay configured for the subscription.
-Once the `retries` are exhausted, the message is `ack()`'ed and re-published to the dead-letter topic.
-The dead-letter topic is the same as the subscription topic, but ends in `.dl`.
+Dead-letter topic is enabled.
+
+An exception or return of `MessageResult.Retry` will `nack()` the message so it is retried by Google up to the specified retries.
+At that point, the message is published to the dead-letter topic and `ack()` is called on the message.
+
+A return of `MessageResult.Fail` will publish the message to the dead-letter topic and `ack()` is called on the message to remove it from the subscription.
+
+The `messageContext` includes `{ attemptsMade: 0-3 }` for each retry. It's possible to be attemptsMade up to 4 due to the race condition above.
+
+#### `{ retries: 4 to 99 }`
+
+Dead-letter topic is enabled.
+
+An exception or return of `MessageResult.Retry` will `nack()` the message so it is retried by Google up to the specified retries.
+
+A return of `MessageResult.Fail` will publish the message to the dead-letter topic and `ack()` is called on the message.
+
+The `messageContext` includes `{ attemptsMade: 0-99 }` for each retry.
+
+#### `{ retries: 100 or more }`
+
+These are not valid when creating a gcp-pubsub subscription. Creating a new subscription will fail.
 
 ### Failing messages
 
-A dead-letter topic is created using the topic name and ends with `.dl`. Messages that fail after the configured `retries` are re-published to the dead-letter topic.
+dead-letter topics are used when:
+
+- `retries` is undefined or greater than or equal to 0
+- or the subscription already has a dead-letter policy in place
+
+dead-letter topics are NOT used when:
+
+- `retries` is less than 0
+- the subscription already exists without a dead-letter policy in place
+
+If a dead-letter topic exists and the message handler returns `MessageResult.Fail` the message will be published to the dead-letter topic and `ack()` will be called on the message, removing it from the subscription.
+
+There is a possible race condition between publishing the dead-letter message and acknowledging the message causing the message to be re-delivered. This can result in the message ending up in the dead-letter topic more than once, or possibly a message in the dead-letter topic and also successfully handled. This race condition only occurs when `@palmetto/pubsub` is publishing to the dead-letter topic. It should not happen when using the gcp-pubsub dead-letter policy.
+
+Note: be sure the dead-letter topic is configured correctly. gcp-pubsub has some strict requirements on how to configure it.
+You can visit the subscription in the Google Cloud console and go to the "Dead lettering" tab to identify and fix any issues.
+The trickiest part might be setting the [IAM roles](https://docs.cloud.google.com/pubsub/docs/dead-letter-topics#grant_forwarding_permissions).
+
+Note: When `@palmetto/pubsub` creates the dead-letter topic, it also creates a default subscription ending in `.sub` to ensure dead-letter messages are not lost.