@palmetto/pubsub 3.1.0 → 3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -75,7 +75,7 @@ class BullMqSubscriber {
75
75
  attemptsMade: job.attemptsMade,
76
76
  firstSent: new Date(job.timestamp),
77
77
  lastSent: undefined,
78
- willRetryOnFailure: this.willRetryJobOnFailure(job),
78
+ willAttemptRetry: this.willRetryJobOnFailure(job),
79
79
  };
80
80
  const result = yield onMessage(job.data, context);
81
81
  if (result === interfaces_js_1.MessageResult.Ok) {
@@ -0,0 +1,66 @@
1
+ import { type CreateSubscriptionOptions, type SubscriptionOptions, type PubSub, Topic } from "@google-cloud/pubsub";
2
+ import { MessageContext, PubSubConfiguration } from "../interfaces.js";
3
+ import { GCP_PUBSUB_TRANSPORT } from "./connection.js";
4
+ export interface GcpPubSubConfiguration extends PubSubConfiguration {
5
+ /**
6
+ * Transport is always GCP_PUBSUB_TRANSPORT
7
+ */
8
+ transport: typeof GCP_PUBSUB_TRANSPORT;
9
+ /**
10
+ * The Pub/Sub topic name. Defaults to `name` if omitted.
11
+ */
12
+ topicName?: string;
13
+ /**
14
+ * The Pub/Sub dead-letter topic name. Defaults to `${name}.dl` if omitted.
15
+ */
16
+ deadLetterTopicName?: string;
17
+ /**
18
+ * The Pub/Sub subscription name. Required for subscribers. Defaults to `name` if omitted.
19
+ */
20
+ subscriptionName?: string;
21
+ /**
22
+ * Flow control: max outstanding messages (default: 5)
23
+ */
24
+ maxMessages?: number;
25
+ /**
26
+ * Maximum number of times to retry a message.
27
+ * Default: 0, no retries
28
+ * Max: 99 retries, see max_delivery_attempts in GCP docs
29
+ * Infinite retries: -1
30
+ */
31
+ retries?: number;
32
+ /**
33
+ * Maximum amount of time to wait, in milliseconds, before retrying a message. Default: 600_000 (10 minutes)
34
+ */
35
+ maxRetryDelay?: number;
36
+ /**
37
+ * Subscription settings to override defaults when creating a new subscription
38
+ */
39
+ createSubscriptionOptions?: CreateSubscriptionOptions;
40
+ /**
41
+ * subscription options to override defaults when starting a new subscription
42
+ */
43
+ subscriptionOptions?: SubscriptionOptions;
44
+ /**
45
+ * During shutdown, pending acks will be waited for up to this amount of time (in milliseconds) after the last message is received before the subscription is closed. Default: 1_000 (1 second)
46
+ */
47
+ shutdownDelay?: number;
48
+ }
49
+ export declare const RETRY_FOREVER = -1;
50
+ export type TopicType = "default" | "dead-letter";
51
+ export declare const TopicNameExtensions: Record<TopicType, string | undefined>;
52
+ export interface GcpPubSubMessageContext extends MessageContext {
53
+ messageId: string;
54
+ subscriptionName: string;
55
+ topicName: string;
56
+ orderingKey?: string;
57
+ }
58
+ export declare function getTopicName(config: GcpPubSubConfiguration, topicType: TopicType): string;
59
+ export declare function getSubscriptionName(config: GcpPubSubConfiguration): string;
60
+ export declare function isGcpPubSubConfiguration(config: PubSubConfiguration): config is GcpPubSubConfiguration;
61
+ export declare function ensureTopic(client: PubSub, topicName: string): Promise<void>;
62
+ export declare function ensureSubscription(client: PubSub, config: GcpPubSubConfiguration): Promise<{
63
+ deadLetterTopic?: Topic;
64
+ retries: number;
65
+ created: boolean;
66
+ }>;
@@ -0,0 +1,126 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.TopicNameExtensions = exports.RETRY_FOREVER = void 0;
13
+ exports.getTopicName = getTopicName;
14
+ exports.getSubscriptionName = getSubscriptionName;
15
+ exports.isGcpPubSubConfiguration = isGcpPubSubConfiguration;
16
+ exports.ensureTopic = ensureTopic;
17
+ exports.ensureSubscription = ensureSubscription;
18
+ const pubsub_1 = require("@google-cloud/pubsub");
19
+ const connection_js_1 = require("./connection.js");
20
+ exports.RETRY_FOREVER = -1;
21
+ exports.TopicNameExtensions = {
22
+ "dead-letter": ".dl",
23
+ default: undefined,
24
+ };
25
+ function getTopicName(config, topicType) {
26
+ var _a, _b;
27
+ if (topicType === "dead-letter" && config.deadLetterTopicName) {
28
+ return config.deadLetterTopicName;
29
+ }
30
+ const baseName = (_a = config.topicName) !== null && _a !== void 0 ? _a : config.name;
31
+ const extension = (_b = exports.TopicNameExtensions[topicType]) !== null && _b !== void 0 ? _b : "";
32
+ return `${baseName}${extension}`;
33
+ }
34
+ function getSubscriptionName(config) {
35
+ var _a;
36
+ return (_a = config.subscriptionName) !== null && _a !== void 0 ? _a : getTopicName(config, "default");
37
+ }
38
+ function isGcpPubSubConfiguration(config) {
39
+ return config.transport === connection_js_1.GCP_PUBSUB_TRANSPORT;
40
+ }
41
+ function ensureTopic(client, topicName) {
42
+ return __awaiter(this, void 0, void 0, function* () {
43
+ const topics = (yield client.getTopics())[0];
44
+ if (!topics.some((t) => t.name.endsWith(`/${topicName}`))) {
45
+ yield client.createTopic(topicName);
46
+ }
47
+ });
48
+ }
49
+ function ensureSubscription(client, config) {
50
+ return __awaiter(this, void 0, void 0, function* () {
51
+ var _a, _b, _c, _d, _e;
52
+ const topic = client.topic(getTopicName(config, "default"));
53
+ const subscriptions = (yield topic.getSubscriptions({
54
+ pageSize: 1000,
55
+ }))[0];
56
+ const subscriptionName = getSubscriptionName(config);
57
+ const subscription = subscriptions.find((s) => s.name.endsWith(`/${subscriptionName}`));
58
+ if (subscription) {
59
+ const meta = (_a = subscription.metadata) !== null && _a !== void 0 ? _a : (yield subscription.getMetadata())[0];
60
+ let deadLetterTopic = undefined;
61
+ let retries;
62
+ if ((_b = meta.deadLetterPolicy) === null || _b === void 0 ? void 0 : _b.deadLetterTopic) {
63
+ const dlp = pubsub_1.protos.google.pubsub.v1.DeadLetterPolicy.create(meta.deadLetterPolicy);
64
+ if (config.retries !== undefined && config.retries < 0) {
65
+ // If retries is negative, we want to retry forever, but GCP Pub/Sub doesn't support that, so we set it to the max allowed by the dead-letter policy
66
+ retries = dlp.maxDeliveryAttempts - 1;
67
+ }
68
+ else {
69
+ // config.retries can override maxDeliveryAttempts, but it can't exceed the max allowed by the dead-letter policy
70
+ retries = Math.min((_c = config.retries) !== null && _c !== void 0 ? _c : 0, dlp.maxDeliveryAttempts - 1);
71
+ }
72
+ deadLetterTopic = client.topic(dlp.deadLetterTopic);
73
+ }
74
+ else {
75
+ retries = exports.RETRY_FOREVER;
76
+ }
77
+ return {
78
+ deadLetterTopic,
79
+ retries,
80
+ created: false,
81
+ };
82
+ }
83
+ const topicName = getTopicName(config, "default");
84
+ const dlTopicName = getTopicName(config, "dead-letter");
85
+ const minRetryDelay = (_d = config.retryDelay) !== null && _d !== void 0 ? _d : 30000;
86
+ const maxRetryDelay = config.maxRetryDelay;
87
+ const maxDeliveryAttempts = ((_e = config.retries) !== null && _e !== void 0 ? _e : 0) + 1;
88
+ const enableDeadLetter = maxDeliveryAttempts > 0;
89
+ if (enableDeadLetter) {
90
+ yield ensureSubscription(client, Object.assign(Object.assign({}, config), { retries: exports.RETRY_FOREVER, name: dlTopicName, topicName: dlTopicName, deadLetterTopicName: undefined, subscriptionName: `${dlTopicName}.sub`, retryDelay: minRetryDelay, maxRetryDelay }));
91
+ }
92
+ yield ensureTopic(client, topicName);
93
+ const enableRetryPolicy = minRetryDelay > 0 || maxRetryDelay !== undefined;
94
+ const deadLetterTopic = enableDeadLetter
95
+ ? client.topic(dlTopicName)
96
+ : undefined;
97
+ const options = Object.assign(Object.assign(Object.assign({}, (enableDeadLetter
98
+ ? {
99
+ deadLetterPolicy: {
100
+ deadLetterTopic: deadLetterTopic === null || deadLetterTopic === void 0 ? void 0 : deadLetterTopic.name,
101
+ maxDeliveryAttempts: Math.max(maxDeliveryAttempts, 5),
102
+ },
103
+ }
104
+ : {})), (enableRetryPolicy
105
+ ? {
106
+ retryPolicy: Object.assign({ minimumBackoff: {
107
+ nanos: (minRetryDelay % 1000) * 1000000,
108
+ seconds: Math.floor(minRetryDelay / 1000),
109
+ } }, (maxRetryDelay !== undefined
110
+ ? {
111
+ maximumBackoff: {
112
+ nanos: (maxRetryDelay % 1000) * 1000000,
113
+ seconds: Math.floor(maxRetryDelay / 1000),
114
+ },
115
+ }
116
+ : {})),
117
+ }
118
+ : {})), config.createSubscriptionOptions);
119
+ yield topic.createSubscription(subscriptionName, options);
120
+ return {
121
+ deadLetterTopic,
122
+ retries: maxDeliveryAttempts > 0 ? maxDeliveryAttempts - 1 : exports.RETRY_FOREVER,
123
+ created: true,
124
+ };
125
+ });
126
+ }
@@ -0,0 +1,14 @@
1
+ import type * as gcppubsub from "@google-cloud/pubsub";
2
+ import { Logger } from "../interfaces.js";
3
+ export declare const GCP_PUBSUB_TRANSPORT = "gcp-pubsub";
4
+ export interface GcpPubSubConnectionConfig {
5
+ projectId?: string;
6
+ clientConfig?: gcppubsub.ClientConfig;
7
+ }
8
+ export declare class GcpPubSubConnection {
9
+ readonly client: gcppubsub.PubSub;
10
+ private readonly logger;
11
+ static create(config: GcpPubSubConnectionConfig, logger: Logger): Promise<GcpPubSubConnection>;
12
+ private constructor();
13
+ close(): Promise<void>;
14
+ }
@@ -0,0 +1,38 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.GcpPubSubConnection = exports.GCP_PUBSUB_TRANSPORT = void 0;
13
+ const lazy_load_js_1 = require("../lazy-load.js");
14
+ exports.GCP_PUBSUB_TRANSPORT = "gcp-pubsub";
15
+ class GcpPubSubConnection {
16
+ static create(config, logger) {
17
+ return __awaiter(this, void 0, void 0, function* () {
18
+ const GcpPubSubPackage = yield (0, lazy_load_js_1.lazyLoad)({
19
+ packageName: "@google-cloud/pubsub",
20
+ context: "GcpPubSubConnection",
21
+ });
22
+ const client = new GcpPubSubPackage.PubSub(Object.assign({ projectId: config.projectId }, config.clientConfig));
23
+ return new GcpPubSubConnection(client, logger);
24
+ });
25
+ }
26
+ constructor(client, logger) {
27
+ this.client = client;
28
+ this.logger = logger;
29
+ }
30
+ close() {
31
+ return __awaiter(this, void 0, void 0, function* () {
32
+ var _a, _b;
33
+ (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, "GCP Pub/Sub closing connection");
34
+ yield this.client.close();
35
+ });
36
+ }
37
+ }
38
+ exports.GcpPubSubConnection = GcpPubSubConnection;
@@ -0,0 +1,5 @@
1
+ export * from "./config.js";
2
+ export * from "./connection.js";
3
+ export * from "./publisher.js";
4
+ export * from "./subscriber.js";
5
+ export * from "./pubsub.js";
@@ -0,0 +1,21 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ __exportStar(require("./config.js"), exports);
18
+ __exportStar(require("./connection.js"), exports);
19
+ __exportStar(require("./publisher.js"), exports);
20
+ __exportStar(require("./subscriber.js"), exports);
21
+ __exportStar(require("./pubsub.js"), exports);
@@ -0,0 +1,14 @@
1
+ import { Logger, PublisherProvider, PublishMessageOptions } from "../interfaces.js";
2
+ import { GcpPubSubConnection } from "./connection.js";
3
+ import { GcpPubSubConfiguration } from "./config.js";
4
+ export declare class GcpPubSubPublisher implements PublisherProvider {
5
+ private readonly connection;
6
+ private readonly logger;
7
+ private readonly topics;
8
+ constructor(connection: GcpPubSubConnection, logger: Logger);
9
+ readonly transport: string;
10
+ init(config: GcpPubSubConfiguration): Promise<void>;
11
+ publish(config: GcpPubSubConfiguration, messages: string[], options?: PublishMessageOptions): Promise<void>;
12
+ enrichPublishedMesssageLog(config: GcpPubSubConfiguration): Record<string, unknown>;
13
+ close(): void;
14
+ }
@@ -0,0 +1,66 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.GcpPubSubPublisher = void 0;
13
+ const errors_js_1 = require("../errors.js");
14
+ const connection_js_1 = require("./connection.js");
15
+ const config_js_1 = require("./config.js");
16
+ class GcpPubSubPublisher {
17
+ constructor(connection, logger) {
18
+ this.connection = connection;
19
+ this.logger = logger;
20
+ this.topics = new Map();
21
+ this.transport = connection_js_1.GCP_PUBSUB_TRANSPORT;
22
+ }
23
+ init(config) {
24
+ return __awaiter(this, void 0, void 0, function* () {
25
+ var _a, _b;
26
+ const topicName = (0, config_js_1.getTopicName)(config, "default");
27
+ yield (0, config_js_1.ensureTopic)(this.connection.client, topicName);
28
+ const topic = this.connection.client.topic(topicName);
29
+ this.topics.set(topicName, topic);
30
+ (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, `GCP Pub/Sub publisher initialized for topic ${topicName}`);
31
+ });
32
+ }
33
+ publish(config, messages, options) {
34
+ return __awaiter(this, void 0, void 0, function* () {
35
+ if (options === null || options === void 0 ? void 0 : options.scheduledAt) {
36
+ throw new errors_js_1.PublishError("Scheduled publishing is not supported for GCP Pub/Sub", messages);
37
+ }
38
+ const topicName = (0, config_js_1.getTopicName)(config, "default");
39
+ let topic = this.topics.get(topicName);
40
+ if (!topic) {
41
+ topic = this.connection.client.topic(topicName);
42
+ this.topics.set(topicName, topic);
43
+ }
44
+ const oks = yield Promise.allSettled(messages.map((message) => topic.publishMessage({
45
+ data: Buffer.from(message, "utf8"),
46
+ })));
47
+ const failedMessages = oks
48
+ .map((ok, index) => ({ ok, index }))
49
+ .filter(({ ok }) => ok.status === "rejected")
50
+ .map(({ index }) => messages[index]);
51
+ if (failedMessages.length > 0) {
52
+ const allOrSome = failedMessages.length === messages.length ? "all" : "some";
53
+ throw new errors_js_1.PublishError(`GCP Pub/Sub publish to topic ${topicName} failed for ${allOrSome} messages`, failedMessages);
54
+ }
55
+ });
56
+ }
57
+ enrichPublishedMesssageLog(config) {
58
+ return {
59
+ topicName: (0, config_js_1.getTopicName)(config, "default"),
60
+ };
61
+ }
62
+ close() {
63
+ this.topics.clear();
64
+ }
65
+ }
66
+ exports.GcpPubSubPublisher = GcpPubSubPublisher;
@@ -0,0 +1,18 @@
1
+ import { Logger, MessageContext, MessageResult, PublishMessageOptions, PubSubProvider, StopSubscribe } from "../interfaces.js";
2
+ import { GcpPubSubConfiguration } from "./config.js";
3
+ import { GcpPubSubConnection } from "./connection.js";
4
+ /**
5
+ * Combines a GcpPubSubPublisher and GcpPubSubSubscriber into a single instance
6
+ */
7
+ export declare class GcpPubSubProvider implements PubSubProvider {
8
+ private readonly publisher;
9
+ private readonly subscriber;
10
+ constructor(connection: GcpPubSubConnection, logger: Logger);
11
+ readonly transport: string;
12
+ publish(config: GcpPubSubConfiguration, messages: string[], options?: PublishMessageOptions): Promise<void>;
13
+ startSubscribe(config: GcpPubSubConfiguration, onMessage: (s: string, context: MessageContext) => Promise<MessageResult> | MessageResult): Promise<StopSubscribe>;
14
+ close(): Promise<void>;
15
+ init(config: GcpPubSubConfiguration): Promise<void>;
16
+ enrichPublishedMesssageLog(config: GcpPubSubConfiguration): Record<string, unknown>;
17
+ enrichHandledMesssageLog(config: GcpPubSubConfiguration): Record<string, unknown>;
18
+ }
@@ -0,0 +1,49 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.GcpPubSubProvider = void 0;
13
+ const connection_js_1 = require("./connection.js");
14
+ const publisher_js_1 = require("./publisher.js");
15
+ const subscriber_js_1 = require("./subscriber.js");
16
+ /**
17
+ * Combines a GcpPubSubPublisher and GcpPubSubSubscriber into a single instance
18
+ */
19
+ class GcpPubSubProvider {
20
+ constructor(connection, logger) {
21
+ this.transport = connection_js_1.GCP_PUBSUB_TRANSPORT;
22
+ this.publisher = new publisher_js_1.GcpPubSubPublisher(connection, logger);
23
+ this.subscriber = new subscriber_js_1.GcpPubSubSubscriber(connection, logger);
24
+ }
25
+ publish(config, messages, options) {
26
+ return this.publisher.publish(config, messages, options);
27
+ }
28
+ startSubscribe(config, onMessage) {
29
+ return this.subscriber.startSubscribe(config, onMessage);
30
+ }
31
+ close() {
32
+ return __awaiter(this, void 0, void 0, function* () {
33
+ yield this.subscriber.close();
34
+ this.publisher.close();
35
+ });
36
+ }
37
+ init(config) {
38
+ return __awaiter(this, void 0, void 0, function* () {
39
+ yield this.publisher.init(config);
40
+ });
41
+ }
42
+ enrichPublishedMesssageLog(config) {
43
+ return this.publisher.enrichPublishedMesssageLog(config);
44
+ }
45
+ enrichHandledMesssageLog(config) {
46
+ return this.subscriber.enrichHandledMesssageLog(config);
47
+ }
48
+ }
49
+ exports.GcpPubSubProvider = GcpPubSubProvider;
@@ -0,0 +1,19 @@
1
+ import { Logger, MessageResult, StopSubscribe, SubscriberProvider } from "../interfaces.js";
2
+ import { GcpPubSubConfiguration, GcpPubSubMessageContext } from "./config.js";
3
+ import { GcpPubSubConnection } from "./connection.js";
4
+ export declare class GcpPubSubSubscriber implements SubscriberProvider {
5
+ private readonly connection;
6
+ private readonly logger;
7
+ private readonly subscriptions;
8
+ private lastMessageDate;
9
+ constructor(connection: GcpPubSubConnection, logger: Logger);
10
+ readonly transport: string;
11
+ startSubscribe(config: GcpPubSubConfiguration, onMessage: (s: string, context: GcpPubSubMessageContext) => Promise<MessageResult> | MessageResult): Promise<StopSubscribe>;
12
+ close(): Promise<void>;
13
+ enrichHandledMesssageLog(config: GcpPubSubConfiguration): Record<string, unknown>;
14
+ private consumeMessage;
15
+ /**
16
+ * Wait for in-flight message handling to complete before shutting down
17
+ */
18
+ private waitForMessagesToComplete;
19
+ }
@@ -0,0 +1,215 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.GcpPubSubSubscriber = void 0;
13
+ const pubsub_1 = require("@google-cloud/pubsub");
14
+ const trace_1 = require("@palmetto/trace");
15
+ const interfaces_js_1 = require("../interfaces.js");
16
+ const errors_js_1 = require("../errors.js");
17
+ const create_log_error_payload_js_1 = require("../create-log-error-payload.js");
18
+ const config_js_1 = require("./config.js");
19
+ const connection_js_1 = require("./connection.js");
20
+ const subscriber_js_1 = require("@google-cloud/pubsub/build/src/subscriber.js");
21
+ const SHUTDOWN_DELAY = 1000;
22
+ class GcpPubSubSubscriber {
23
+ constructor(connection, logger) {
24
+ this.connection = connection;
25
+ this.logger = logger;
26
+ this.subscriptions = new Map();
27
+ this.lastMessageDate = 0;
28
+ this.transport = connection_js_1.GCP_PUBSUB_TRANSPORT;
29
+ }
30
+ startSubscribe(config, onMessage) {
31
+ return __awaiter(this, void 0, void 0, function* () {
32
+ var _a, _b, _c, _d;
33
+ const subscriptionName = (0, config_js_1.getSubscriptionName)(config);
34
+ if (this.subscriptions.has(subscriptionName)) {
35
+ throw new errors_js_1.AlreadySubscribingError(subscriptionName);
36
+ }
37
+ const topicName = (0, config_js_1.getTopicName)(config, "default");
38
+ const { deadLetterTopic, retries } = yield (0, config_js_1.ensureSubscription)(this.connection.client, config);
39
+ if (retries !== ((_a = config.retries) !== null && _a !== void 0 ? _a : 0)) {
40
+ const configRetries = (_b = config.retries) !== null && _b !== void 0 ? _b : 0;
41
+ const log = {
42
+ message: `GCP Pub/Sub subscription ${subscriptionName} retry configuration mismatch.`,
43
+ extra: {
44
+ subscriptionName,
45
+ configRetries,
46
+ subscriptionRetries: retries,
47
+ },
48
+ };
49
+ this.logger.warn(log);
50
+ }
51
+ const subscription = this.connection.client.subscription(subscriptionName, Object.assign({ flowControl: {
52
+ maxMessages: (_c = config.maxMessages) !== null && _c !== void 0 ? _c : 5,
53
+ }, closeOptions: {
54
+ behavior: subscriber_js_1.SubscriberCloseBehaviors.WaitForProcessing,
55
+ timeout: pubsub_1.Duration.from({
56
+ milliseconds: (_d = config.shutdownDelay) !== null && _d !== void 0 ? _d : SHUTDOWN_DELAY,
57
+ }),
58
+ } }, config.subscriptionOptions));
59
+ const messageHandler = (message) => {
60
+ void this.consumeMessage({
61
+ config,
62
+ message,
63
+ onMessage,
64
+ subscriptionName,
65
+ topicName,
66
+ deadLetterTopic,
67
+ retries,
68
+ });
69
+ };
70
+ const errorHandler = (err) => {
71
+ const log = {
72
+ message: `GCP Pub/Sub subscriber error for ${subscriptionName}`,
73
+ error: (0, create_log_error_payload_js_1.createLogErrorPayload)(err),
74
+ };
75
+ this.logger.error(log);
76
+ };
77
+ subscription.on("message", messageHandler);
78
+ subscription.on("error", errorHandler);
79
+ this.subscriptions.set(subscriptionName, { subscription, config });
80
+ this.logger.log(`GCP Pub/Sub consumer started for ${subscriptionName}`);
81
+ return () => __awaiter(this, void 0, void 0, function* () {
82
+ subscription.removeListener("message", messageHandler);
83
+ subscription.removeListener("error", errorHandler);
84
+ yield this.waitForMessagesToComplete(config.shutdownDelay);
85
+ yield subscription.close();
86
+ this.subscriptions.delete(subscriptionName);
87
+ this.logger.log(`GCP Pub/Sub consumer stopped for ${subscriptionName}`);
88
+ });
89
+ });
90
+ }
91
+ close() {
92
+ return __awaiter(this, void 0, void 0, function* () {
93
+ yield Promise.all([...this.subscriptions.entries()].map((_a) => __awaiter(this, [_a], void 0, function* ([subscriptionName, sub]) {
94
+ yield this.waitForMessagesToComplete(sub.config.shutdownDelay);
95
+ yield sub.subscription.close();
96
+ this.logger.log(`GCP Pub/Sub consumer stopped for ${subscriptionName}`);
97
+ })));
98
+ this.subscriptions.clear();
99
+ });
100
+ }
101
+ enrichHandledMesssageLog(config) {
102
+ return {
103
+ subscriptionName: (0, config_js_1.getSubscriptionName)(config),
104
+ topicName: (0, config_js_1.getTopicName)(config, "default"),
105
+ };
106
+ }
107
+ consumeMessage(_a) {
108
+ return __awaiter(this, arguments, void 0, function* ({ config, message, onMessage, subscriptionName, topicName, deadLetterTopic, retries, }) {
109
+ yield (0, trace_1.getTracer)().trace("pubsub.gcppubsub.consume", {
110
+ resource: `consume ${config.transport} ${subscriptionName}`,
111
+ }, (span) => __awaiter(this, void 0, void 0, function* () {
112
+ var _a, _b, _c, _d, _e, _f;
113
+ try {
114
+ const firstPublished = message.publishTime;
115
+ let retryCount = message.deliveryAttempt - 1;
116
+ let willAttemptRetry;
117
+ if (retryCount < 0 || retries === config_js_1.RETRY_FOREVER) {
118
+ // in this case, we will retry forever
119
+ retryCount = 0;
120
+ willAttemptRetry = true;
121
+ }
122
+ else {
123
+ willAttemptRetry = retryCount < retries;
124
+ }
125
+ const context = {
126
+ attemptsMade: retryCount,
127
+ firstSent: firstPublished,
128
+ lastSent: undefined,
129
+ willAttemptRetry,
130
+ messageId: message.id,
131
+ subscriptionName,
132
+ topicName,
133
+ orderingKey: message.orderingKey,
134
+ };
135
+ let messageResult;
136
+ let logPayload;
137
+ try {
138
+ messageResult = yield onMessage(message.data.toString("utf8"), context);
139
+ logPayload = {
140
+ message: `GCP Pub/Sub consumer handled message.`,
141
+ extra: Object.assign({}, context),
142
+ };
143
+ }
144
+ catch (err) {
145
+ messageResult = interfaces_js_1.MessageResult.Retry;
146
+ logPayload = {
147
+ message: `GCP Pub/Sub consumer unhandled exception.`,
148
+ error: (0, create_log_error_payload_js_1.createLogErrorPayload)(err),
149
+ extra: Object.assign({}, context),
150
+ };
151
+ }
152
+ this.lastMessageDate = Date.now();
153
+ if (messageResult === interfaces_js_1.MessageResult.Ok) {
154
+ (_b = (_a = this.logger).debug) === null || _b === void 0 ? void 0 : _b.call(_a, logPayload);
155
+ message.ack();
156
+ return;
157
+ }
158
+ if (messageResult === interfaces_js_1.MessageResult.Retry && willAttemptRetry) {
159
+ message.nack();
160
+ logPayload.message += " Retrying message via gcp-pubsub.";
161
+ (_d = (_c = this.logger).debug) === null || _d === void 0 ? void 0 : _d.call(_c, logPayload);
162
+ return;
163
+ }
164
+ // Fail: either explicit Fail or Retry with no retries remaining
165
+ if (messageResult === interfaces_js_1.MessageResult.Retry) {
166
+ logPayload.message += " No more retries.";
167
+ }
168
+ (_f = (_e = this.logger).debug) === null || _f === void 0 ? void 0 : _f.call(_e, logPayload);
169
+ if (deadLetterTopic &&
170
+ (messageResult === interfaces_js_1.MessageResult.Fail || retryCount < 4)) {
171
+ // Send all failed messages to the dead-letter topic
172
+ // GCP Pub/Sub doesn't allow maxDeliveryAttempts less than 5, so we need to manually publish to the dead-letter topic for the first 4 retries
173
+ logPayload.message += " Publishing message to dead-letter topic.";
174
+ yield deadLetterTopic.publishMessage({
175
+ data: message.data,
176
+ });
177
+ message.ack(); // ack the message to remove it from the subscription
178
+ }
179
+ else if (messageResult === interfaces_js_1.MessageResult.Fail) {
180
+ message.ack(); // ack the message to remove it from the subscription
181
+ }
182
+ else {
183
+ message.nack(); // nack the message to let pubsub send it to the DLQ
184
+ }
185
+ }
186
+ catch (err) {
187
+ const logPayload = {
188
+ message: "Unexpected error handling GCP Pub/Sub message",
189
+ error: (0, create_log_error_payload_js_1.createLogErrorPayload)(err),
190
+ extra: {
191
+ subscriptionName,
192
+ message: message.data.toString("utf8"),
193
+ },
194
+ };
195
+ this.logger.error(logPayload);
196
+ span === null || span === void 0 ? void 0 : span.setTag("error", err);
197
+ message.nack();
198
+ }
199
+ }));
200
+ });
201
+ }
202
+ /**
203
+ * Wait for in-flight message handling to complete before shutting down
204
+ */
205
+ waitForMessagesToComplete() {
206
+ return __awaiter(this, arguments, void 0, function* (shutdownDelay = SHUTDOWN_DELAY) {
207
+ const waitUntil = this.lastMessageDate + shutdownDelay;
208
+ const waitDuration = waitUntil - Date.now();
209
+ if (waitDuration > 0) {
210
+ yield new Promise((resolve) => setTimeout(resolve, waitDuration));
211
+ }
212
+ });
213
+ }
214
+ }
215
+ exports.GcpPubSubSubscriber = GcpPubSubSubscriber;
@@ -66,15 +66,17 @@ export interface PubSubConfiguration {
66
66
  }
67
67
  export declare enum MessageResult {
68
68
  /**
69
- * the message was handled successfully or can otherwise be ignored
69
+ * The message was handled. The transport should not deliver this message again.
70
70
  */
71
71
  Ok = "ok",
72
72
  /**
73
- * The message should be retried
73
+ * The message should be retried by the transport.
74
+ * Whether the message is retried immediately or after a delay depends on the transport and configuration.
75
+ * If the number of retries exceeds the configured limit for the transport, the message is considered failed.
74
76
  */
75
77
  Retry = "retry",
76
78
  /**
77
- * The message failed and is either discarded or stored in a dead-letter queue
79
+ * The message failed and is either discarded or stored in a dead-letter queue, depending on the transport and configuration.
78
80
  */
79
81
  Fail = "fail"
80
82
  }
@@ -129,8 +131,23 @@ export type PubSubProvider = PublisherProvider & SubscriberProvider;
129
131
  */
130
132
  export type PubOrSubProvider = PubSubProvider | PublisherProvider | SubscriberProvider;
131
133
  export interface MessageContext {
134
+ /**
135
+ * When the message was first published.
136
+ */
132
137
  firstSent: Date | undefined;
138
+ /**
139
+ * When the message was last published. Not all transports provide this information.
140
+ */
133
141
  lastSent: Date | undefined;
142
+ /**
143
+ * The number of times the message has been attempted to be processed. This is 0 for the first attempt, 1 for the first retry, etc.
144
+ * Note that for some transports, this number is an approximation based on the number of times the message has been redelivered, or may always be 0.
145
+ */
134
146
  attemptsMade: number | undefined;
135
- willRetryOnFailure: boolean;
147
+ /**
148
+ * If the onMessage callback returns MessageResult.Retry or throws an exception:
149
+ * When true, the message will be retried
150
+ * When false, the message will not be retried
151
+ */
152
+ willAttemptRetry: boolean;
136
153
  }
@@ -14,15 +14,17 @@ exports.IdMetaSchema = zod_1.z.object({
14
14
  var MessageResult;
15
15
  (function (MessageResult) {
16
16
  /**
17
- * the message was handled successfully or can otherwise be ignored
17
+ * The message was handled. The transport should not deliver this message again.
18
18
  */
19
19
  MessageResult["Ok"] = "ok";
20
20
  /**
21
- * The message should be retried
21
+ * The message should be retried by the transport.
22
+ * Whether the message is retried immediately or after a delay depends on the transport and configuration.
23
+ * If the number of retries exceeds the configured limit for the transport, the message is considered failed.
22
24
  */
23
25
  MessageResult["Retry"] = "retry";
24
26
  /**
25
- * The message failed and is either discarded or stored in a dead-letter queue
27
+ * The message failed and is either discarded or stored in a dead-letter queue, depending on the transport and configuration.
26
28
  */
27
29
  MessageResult["Fail"] = "fail";
28
30
  })(MessageResult || (exports.MessageResult = MessageResult = {}));
package/dist/main.d.ts CHANGED
@@ -4,3 +4,4 @@ export * from "./publisher.js";
4
4
  export * from "./subscriber.js";
5
5
  export * from "./bullmq/main.js";
6
6
  export * from "./rabbitmq/main.js";
7
+ export * from "./gcppubsub/main.js";
package/dist/main.js CHANGED
@@ -20,3 +20,4 @@ __exportStar(require("./publisher.js"), exports);
20
20
  __exportStar(require("./subscriber.js"), exports);
21
21
  __exportStar(require("./bullmq/main.js"), exports);
22
22
  __exportStar(require("./rabbitmq/main.js"), exports);
23
+ __exportStar(require("./gcppubsub/main.js"), exports);
@@ -1,6 +1,6 @@
1
1
  import { Logger, MessageLogLevel } from "./interfaces";
2
- export declare function startTiming(): [number, number];
3
- export declare function getDuration(start: [number, number]): number;
2
+ export declare function startTiming(): bigint;
3
+ export declare function getDuration(start: bigint): number;
4
4
  export declare function logMessage({ note, message, logger, level, extra, }: {
5
5
  note: string;
6
6
  message: unknown;
@@ -4,10 +4,10 @@ exports.startTiming = startTiming;
4
4
  exports.getDuration = getDuration;
5
5
  exports.logMessage = logMessage;
6
6
  function startTiming() {
7
- return process.hrtime();
7
+ return process.hrtime.bigint();
8
8
  }
9
9
  function getDuration(start) {
10
- return process.hrtime(start)[1] / 1000000; // convert to milliseconds
10
+ return Number(process.hrtime.bigint() - start) / 1000000; // convert to milliseconds
11
11
  }
12
12
  function logMessage({ note, message, logger, level = "debug", extra, }) {
13
13
  var _a;
@@ -195,8 +195,8 @@ class RabbitMqSubscriber {
195
195
  const json = msg.content.toString("utf8");
196
196
  const attemptsMade = RabbitMqSubscriber.getAttemptsMade(msg);
197
197
  const sentDates = RabbitMqSubscriber.getSentDates(msg);
198
- const willRetryOnFailure = attemptsMade < ((_b = subscribedMessage.config.retries) !== null && _b !== void 0 ? _b : 0);
199
- const context = Object.assign(Object.assign({ attemptsMade }, sentDates), { queueName, exchangeName: msg.fields.exchange, retryExchangeName, routingKey: msg.fields.routingKey, willRetryOnFailure });
198
+ const willAttemptRetry = attemptsMade < ((_b = subscribedMessage.config.retries) !== null && _b !== void 0 ? _b : 0);
199
+ const context = Object.assign(Object.assign({ attemptsMade }, sentDates), { queueName, exchangeName: msg.fields.exchange, retryExchangeName, routingKey: msg.fields.routingKey, willAttemptRetry });
200
200
  let messageResult;
201
201
  let logPayload;
202
202
  try {
@@ -220,7 +220,7 @@ class RabbitMqSubscriber {
220
220
  return;
221
221
  }
222
222
  if (messageResult === interfaces_js_1.MessageResult.Retry) {
223
- if (willRetryOnFailure) {
223
+ if (willAttemptRetry) {
224
224
  logPayload.message += " Retrying message.";
225
225
  }
226
226
  else {
@@ -108,13 +108,13 @@ class Subscriber {
108
108
  (0, message_logger_js_1.logMessage)({
109
109
  note: "Subscriber error when processing message",
110
110
  message: decodeResult.data,
111
- level: context.willRetryOnFailure ? "warn" : "error",
111
+ level: context.willAttemptRetry ? "warn" : "error",
112
112
  logger: this.logger,
113
113
  extra: Object.assign(Object.assign({ transport: provider.transport, name: config.name, durationMs,
114
114
  context }, enrichedConfig), { error: (0, create_log_error_payload_js_1.createLogErrorPayload)(err) }),
115
115
  });
116
- // in the messageHandled event, we always log as retry or fail based on willRetryOnFailure
117
- const result = context.willRetryOnFailure
116
+ // in the messageHandled event, we always log as retry or fail based on willAttemptRetry
117
+ const result = context.willAttemptRetry
118
118
  ? interfaces_js_1.MessageResult.Retry
119
119
  : interfaces_js_1.MessageResult.Fail;
120
120
  const eventContext = {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@palmetto/pubsub",
3
- "version": "3.1.0",
3
+ "version": "3.2.1",
4
4
  "repository": {
5
5
  "type": "git",
6
6
  "url": "https://github.com/palmetto/galaxy"
@@ -24,6 +24,7 @@
24
24
  "test:watch": "yarn run test-runner vitest watch"
25
25
  },
26
26
  "devDependencies": {
27
+ "@google-cloud/pubsub": "^5.2.3",
27
28
  "@palmetto/trace": "^0.1.0",
28
29
  "@types/amqplib": "^0",
29
30
  "@types/node": "^24.2.1",
@@ -49,10 +50,11 @@
49
50
  "uuid": "^11.1.0"
50
51
  },
51
52
  "peerDependencies": {
53
+ "@google-cloud/pubsub": "^5.2",
52
54
  "@palmetto/trace": "^0.1.0",
53
55
  "amqp-connection-manager": "^4.1.14",
54
56
  "amqplib": "^0.10.8",
55
57
  "bullmq": "^5.58.0",
56
- "zod": "^4.1.13"
58
+ "zod": "^4.1"
57
59
  }
58
60
  }
@@ -0,0 +1,216 @@
1
+ # @palmetto/pubsub
2
+
3
+ The GoogleCloud Pub/Sub (gcp-pubsub) transport provider for @palmetto/pubsub
4
+
5
+ ## Installation
6
+
7
+ ```sh
8
+ yarn add @palmetto/pubsub @google-cloud/pubsub zod
9
+ ```
10
+
11
+ ## Usage
12
+
13
+ 1. Define your connection string
14
+
15
+ ```ts
16
+ import { GcpPubSubConnectionConfig } from "@palmetto/pubsub";
17
+
18
+ const config: GcpPubSubConnectionConfig = {
19
+ projectId: "GOOGLE_PROJECT_ID",
20
+ };
21
+ ```
22
+
23
+ 2. Create the connection
24
+
25
+ ```ts
26
+ import { GcpPubSubConnection } from "@palmetto/pubsub";
27
+
28
+ const connection = await GcpPubSubConnection.create(config, console);
29
+ ```
30
+
31
+ 3. Create the publisher
32
+
33
+ ```ts
34
+ import { Publisher, GcpPubSubPublisher } from "@palmetto/pubsub";
35
+
36
+ const publisher = new Publisher(console, [
37
+ new GcpPubSubPublisher(connection, console),
38
+ ]);
39
+ ```
40
+
41
+ 4. Create the subscriber
42
+
43
+ ```ts
44
+ import { Subscriber, GcpPubSubSubscriber } from "@palmetto/pubsub";
45
+
46
+ const subscriber = new Subscriber(console, [
47
+ new GcpPubSubSubscriber(connection, console),
48
+ ]);
49
+ ```
50
+
51
+ 5. Create the topic/subscriber configuration
52
+
53
+ ```ts
54
+ import {
55
+ GcpPubSubConfiguration,
56
+ GCP_PUBSUB_TRANSPORT,
57
+ } from "@palmetto/pubsub";
58
+
59
+ import { z } from "zod";
60
+
61
+ const MyModelSchema = z.object({...});
62
+
63
+ const config: GcpPubSubConfiguration = {
64
+ name: "my-name",
65
+ topicName: "my-topic-name", // default: name
66
+ subscriptionName: "my-subscription-name", // default: topicName or name
67
+ schema: MyModelSchema,
68
+ transport: GCP_PUBSUB_TRANSPORT,
69
+ retries: 10, // Note: enables dead-letter policy. Use RETRY_FOREVER to disable dead-letter. @palmetto/pubsub will not update an existing subscription.
70
+ retryDelay: 1_000, // Note: retryDelay is minimum retry milliseconds using the exponential backoff retry policy. @palmetto/pubsub will not update an existing subscription.
71
+ createSubscriptionOptions: { ... }, // set the specific createSubscription() options you want to use [optional]
72
+ subscriptionOptions: { ... }, // set any specific subscribe() options you want to use [optional]
73
+ };
74
+ ```
75
+
76
+ 6. Initialize the publisher
77
+
78
+ For performance reasons, it's a good idea to initialize the publisher on your API startup. This can reduce the latency for the first message published.
79
+
80
+ ```ts
81
+ await publisher.init(config);
82
+ ```
83
+
84
+ 7. Publish a message
85
+
86
+ ```ts
87
+ const message: MyModel = {... };
88
+
89
+ await publisher.publish(config, message);
90
+ ```
91
+
92
+ 8. Subscribe to a topic
93
+
94
+ ```ts
95
+ import { GcpPubSubMessageContext } from "@palmetto/pubsub";
96
+
97
+ subscriber.addSubscriber(queue, (message: MyModel, context: GcpPubSubMessageContext) => {
98
+ ...
99
+ return MessageResult.Ok;
100
+ });
101
+ ```
102
+
103
+ 9. Subscribe to the dead-letter topic if you want to re-process completely failed messages. Pass the dead-letter topicName to the config.
104
+
105
+ ```ts
106
+ import { getTopicName } from "@palmetto/pubsub";
107
+
108
+ const dlConfig = {
109
+ ...config,
110
+ topicName: getTopicName(config, "dead-letter")
111
+ };
112
+
113
+ subscriber.addSubscriber(dlConfig, (message: MyModel, context: GcpPubSubMessageContext) => {
114
+ ...
115
+ return MessageResult.Ok; // returning Ok will remove the message from the dead-letter topic
116
+ });
117
+ ```
118
+
119
+ ## Message Handlers
120
+
121
+ Message handlers can return one of 3 values:
122
+
123
+ - `MessageResult.Ok` - the message can be removed from subscription. Messages are always `ack()`'ed.
124
+ - `MessageResult.Retry` - the message can be retried, depending on the dead-letter/retry configuration. Messages are typically `nack()`'ed to use gcp-pubsub dead-letter policy. [See below](#retrying-messages)
125
+ - `MessageResult.Fail` - Remove it from the subscription and publish it to the dead-letter topic (if available). Messages are always `ack()`'ed. [See below](#failing-messages)
126
+
127
+ If a message handler throws an exception, this is the same as if it returned `MessageResult.Retry`.
128
+
129
+ ### Retrying messages
130
+
131
+ By default gcp-pubsub subscriptions retry messages immediately, but `@palmetto/pubsub` subscriptions default to a 30 second retry policy.
132
+
133
+ - minimum backoff: default 30 seconds
134
+ - maximum backoff: default 10 minutes (gcp-pubsub default)
135
+
136
+ You can adjust these defaults using these two configuration settings:
137
+
138
+ - `retryDelay`
139
+ - 0 : immediate retry
140
+ - 1 to 600_000 : enable exponential backoff retry policy (values over 600_000 [10 minutes] are not supported by gcp-pubsub)
141
+ - `maxRetryDelay`
142
+ - any value here enables exponential backoff retry policy
143
+ - be sure this value is equal to or larger than the `retryDelay`
144
+
145
+ Messages can be retried when the handler throws an exception or when it returns `MessageResult.Retry`.
146
+
147
+ The `retries` property configures the subscription retry policy when `@palmetto/pubsub` creates the subscription. If `@palmetto/pubsub` didn't create the subscription, whatever retry policy in place is what will be used (by default there are infinite retries).
148
+
149
+ NOTE: If the subscription already exists, and there is no dead-letter policy, the configured `retries` value is ignored and assumes `RETRY_FOREVER` instead.
150
+
151
+ The following special values define how `@palmetto/pubsub` creates the subscription retry policy:
152
+
153
+ #### `{ retries: RETRY_FOREVER }`
154
+
155
+ Use `RETRY_FOREVER` or `-1` to disable dead-letter policy.
156
+
157
+ When a message handler throws an exception or returns `MessageResult.Retry` the message is retried. Messages are retried until the message retention period expires.
158
+
159
+ When a message handler returns `MessageResult.Fail` or `MessageResult.Ok`, `ack()` is called on the message to remove it from the subscription.
160
+
161
+ The `messageContext` includes `{ attemptsMade: 0 }` on every message.
162
+
163
+ #### `{ retries: undefined }` or `{ retries: 0 }`
164
+
165
+ There are no retries, but dead-letter topic is enabled.
166
+
167
+ An exception or return values of `MessageResult.Fail` or `MessageResult.Retry` the message is published to the dead-letter topic and `ack()` is called on the message to remove it from the subscription.
168
+
169
+ The `messageContext` includes `{ attemptsMade: 0 }` on the first message and will be a number greater than 0 if the message is retried due to the race condition.
170
+
171
+ #### `{ retries: 1, 2 or 3 }`
172
+
173
+ Dead-letter topic is enabled.
174
+
175
+ An exception or return of `MessageResult.Retry` will `nack()` the message so it is retried by Google up to the specified retries.
176
+ At that point, the message is published to the dead-letter topic and `ack()` is called on the message.
177
+
178
+ A return of `MessageResult.Fail` will publish the message to the dead-letter topic and `ack()` is called on the message to remove it from the subscription.
179
+
180
+ The `messageContext` includes `{ attemptsMade: 0-3 }` for each retry. It's possible to be attemptsMade up to 4 due to the race condition above.
181
+
182
+ #### `{ retries: 4 to 99 }`
183
+
184
+ Dead-letter topic is enabled.
185
+
186
+ An exception or return of `MessageResult.Retry` will `nack()` the message so it is retried by Google up to the specified retries.
187
+
188
+ A return of `MessageResult.Fail` will publish the message to the dead-letter topic and `ack()` is called on the message.
189
+
190
+ The `messageContext` includes `{ attemptsMade: 0-99 }` for each retry.
191
+
192
+ #### `{ retries: 100 or more }`
193
+
194
+ These are not valid when creating a gcp-pubsub subscription. Creating a new subscription will fail.
195
+
196
+ ### Failing messages
197
+
198
+ dead-letter topics are used when:
199
+
200
+ - `retries` is undefined or greater than or equal to 0
201
+ - or the subscription already has a dead-letter policy in place
202
+
203
+ dead-letter topics are NOT used when:
204
+
205
+ - `retries` is less than 0
206
+ - the subscription already exists without a dead-letter policy in place
207
+
208
+ If a dead-letter topic exists and the message handler returns `MessageResult.Fail` the message will be published to the dead-letter topic and `ack()` will be called on the message, removing it from the subscription.
209
+
210
+ There is a possible race condition between publishing the dead-letter message and acknowledging the message causing the message to be re-delivered. This can result in the message ending up in the dead-letter topic more than once, or possibly a message in the dead-letter topic and also successfully handled. This race condition only occurs when `@palmetto/pubsub` is publishing to the dead-letter topic. It should not happen when using the gcp-pubsub dead-letter policy.
211
+
212
+ Note: be sure the dead-letter topic is configured correctly. gcp-pubsub has some strict requirements on how to configure it.
213
+ You can visit the subscription in the Google Cloud console and go to the "Dead lettering" tab to identify and fix any issues.
214
+ The trickiest part might be setting the [IAM roles](https://docs.cloud.google.com/pubsub/docs/dead-letter-topics#grant_forwarding_permissions).
215
+
216
+ Note: When `@palmetto/pubsub` creates the dead-letter topic, it also creates a default subscription ending in `.sub` to ensure dead-letter messages are not lost.
@@ -13,6 +13,8 @@ yarn add @palmetto/pubsub amqp-connection-manager amqplib zod
13
13
  1. Define your connection string
14
14
 
15
15
  ```ts
16
+ import { RabbitMqConnectionConfig } from "@palmetto/pubsub";
17
+
16
18
  const config: RabbitMqConnectionConfig = {
17
19
  host: "amqp://guest:guest@localhost:5672/",
18
20
  };
@@ -21,20 +23,26 @@ yarn add @palmetto/pubsub amqp-connection-manager amqplib zod
21
23
  2. Create the connection
22
24
 
23
25
  ```ts
24
- const connection = new RabbitMqConnection(config, console);
26
+ import { RabbitMqConnection } from "@palmetto/pubsub";
27
+
28
+ const connection = await RabbitMqConnection.create(config, console);
25
29
  ```
26
30
 
27
31
  3. Create the publisher
28
32
 
29
33
  ```ts
34
+ import { Publisher, RabbitMqPublisher } from "@palmetto/pubsub";
35
+
30
36
  const publisher = new Publisher(console, [
31
- RabbitMqPublisher(connection, console),
37
+ new RabbitMqPublisher(connection, console),
32
38
  ]);
33
39
  ```
34
40
 
35
41
  4. Create the subscriber
36
42
 
37
43
  ```ts
44
+ import { Subscriber, RabbitMqSubscriber } from "@palmetto/pubsub";
45
+
38
46
  const subscriber = new Subscriber(console, [
39
47
  new RabbitMqSubscriber(connection, console),
40
48
  ]);
@@ -43,6 +51,11 @@ yarn add @palmetto/pubsub amqp-connection-manager amqplib zod
43
51
  5. Create the queue configuration
44
52
 
45
53
  ```ts
54
+ import { RabbitQueueExchangeConfiguration } from "@palmetto/pubsub";
55
+ import { z } from "zod";
56
+
57
+ const MyModelSchema = z.object({...});
58
+
46
59
  const queue: RabbitQueueExchangeConfiguration = {
47
60
  name: "my-queue-name",
48
61
  schema: MyModelSchema,
@@ -101,7 +114,7 @@ Message failures in RabbitMq subscribers are handled using special queues and ex
101
114
 
102
115
  These properties configure message retries:
103
116
 
104
- - `retries` : specifies the number of retries after which the message is discarded. When this value is undefined, the message will retry forever.
117
+ - `retries` : specifies the number of retries after which the message is discarded. When this value is undefined, the message will not retry.
105
118
  - `retryDelay`: specifies the number of milliseconds between retries.
106
119
 
107
120
  #### The retry queue