@palmetto/pubsub 3.2.2 → 3.2.4

package/README.md

@@ -55,7 +55,7 @@ yarn add @palmetto/pubsub zod @palmetto/trace dd-trace
    const publisher = new Publisher(console, [rabbitMqPublisher]);
 
    const message: Model = {
-     id: undefined,
+     id: undefined, // leave id and meta undefined, the publisher will set these
      meta: undefined,
      message: "Hello, world!",
    };
@@ -145,6 +145,10 @@ For specific details about using RabbitMQ see [RabbitMQ README.md](src/rabbitmq/
 
 For specific details about using BullMQ see [BullMQ README.md](src/bullmq/README.md)
 
+### Google Cloud Pubsub Usage
+
+For specific details about using Google Cloud PubSub see [gcppubsub README.md](src/bcppubsub/README.md)
+
 ### Defining schemas
 
 All schemas must be defined using `zod` v4.1+. By convention, all messages should extend the `IdMetaSchema` schema, but it is not strictly required.

package/dist/gcppubsub/config.d.ts

@@ -1,4 +1,4 @@
-import { type CreateSubscriptionOptions, type SubscriptionOptions, type PubSub, Topic } from "@google-cloud/pubsub";
+import type * as gcppubsub from "@google-cloud/pubsub";
 import { MessageContext, PubSubConfiguration } from "../interfaces.js";
 import { GCP_PUBSUB_TRANSPORT } from "./connection.js";
 export interface GcpPubSubConfiguration extends PubSubConfiguration {
@@ -36,11 +36,11 @@ export interface GcpPubSubConfiguration extends PubSubConfiguration {
     /**
      * Subscription settings to override defaults when creating a new subscription
      */
-    createSubscriptionOptions?: CreateSubscriptionOptions;
+    createSubscriptionOptions?: gcppubsub.CreateSubscriptionOptions;
     /**
      * subscription options to override defaults when starting a new subscription
      */
-    subscriptionOptions?: SubscriptionOptions;
+    subscriptionOptions?: gcppubsub.SubscriptionOptions;
     /**
      * During shutdown, pending acks will be waited for up to this amount of time (in milliseconds) after the last message is received before the subscription is closed. Default: 1_000 (1 second)
      */
@@ -58,9 +58,9 @@ export interface GcpPubSubMessageContext extends MessageContext {
 export declare function getTopicName(config: GcpPubSubConfiguration, topicType: TopicType): string;
 export declare function getSubscriptionName(config: GcpPubSubConfiguration): string;
 export declare function isGcpPubSubConfiguration(config: PubSubConfiguration): config is GcpPubSubConfiguration;
-export declare function ensureTopic(client: PubSub, topicName: string): Promise<void>;
-export declare function ensureSubscription(client: PubSub, config: GcpPubSubConfiguration): Promise<{
-    deadLetterTopic?: Topic;
+export declare function ensureTopic(client: gcppubsub.PubSub, topicName: string): Promise<void>;
+export declare function ensureSubscription(client: gcppubsub.PubSub, config: GcpPubSubConfiguration): Promise<{
+    deadLetterTopic?: gcppubsub.Topic;
     retries: number;
     created: boolean;
 }>;

package/dist/gcppubsub/config.js

@@ -15,7 +15,6 @@ exports.getSubscriptionName = getSubscriptionName;
 exports.isGcpPubSubConfiguration = isGcpPubSubConfiguration;
 exports.ensureTopic = ensureTopic;
 exports.ensureSubscription = ensureSubscription;
-const pubsub_1 = require("@google-cloud/pubsub");
 const connection_js_1 = require("./connection.js");
 exports.RETRY_FOREVER = -1;
 exports.TopicNameExtensions = {
@@ -48,7 +47,7 @@ function ensureTopic(client, topicName) {
 }
 function ensureSubscription(client, config) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a, _b, _c, _d, _e;
+        var _a, _b, _c, _d, _e, _f;
         const topic = client.topic(getTopicName(config, "default"));
         const subscriptions = (yield topic.getSubscriptions({
             pageSize: 1000,
@@ -60,16 +59,17 @@ function ensureSubscription(client, config) {
             let deadLetterTopic = undefined;
             let retries;
             if ((_b = meta.deadLetterPolicy) === null || _b === void 0 ? void 0 : _b.deadLetterTopic) {
-                const dlp = pubsub_1.protos.google.pubsub.v1.DeadLetterPolicy.create(meta.deadLetterPolicy);
+                const maxDeliveryAttempts = (_c = meta.deadLetterPolicy.maxDeliveryAttempts) !== null && _c !== void 0 ? _c : 0;
+                const deadLetterTopicName = meta.deadLetterPolicy.deadLetterTopic || "";
                 if (config.retries !== undefined && config.retries < 0) {
                     // If retries is negative, we want to retry forever, but GCP Pub/Sub doesn't support that, so we set it to the max allowed by the dead-letter policy
-                    retries = dlp.maxDeliveryAttempts - 1;
+                    retries = maxDeliveryAttempts - 1;
                 }
                 else {
                     // config.retries can override maxDeliveryAttempts, but it can't exceed the max allowed by the dead-letter policy
-                    retries = Math.min((_c = config.retries) !== null && _c !== void 0 ? _c : 0, dlp.maxDeliveryAttempts - 1);
+                    retries = Math.min((_d = config.retries) !== null && _d !== void 0 ? _d : 0, maxDeliveryAttempts - 1);
                 }
-                deadLetterTopic = client.topic(dlp.deadLetterTopic);
+                deadLetterTopic = client.topic(deadLetterTopicName);
             }
             else {
                 retries = exports.RETRY_FOREVER;
@@ -82,9 +82,9 @@ function ensureSubscription(client, config) {
         }
         const topicName = getTopicName(config, "default");
         const dlTopicName = getTopicName(config, "dead-letter");
-        const minRetryDelay = (_d = config.retryDelay) !== null && _d !== void 0 ? _d : 30000;
+        const minRetryDelay = (_e = config.retryDelay) !== null && _e !== void 0 ? _e : 30000;
         const maxRetryDelay = config.maxRetryDelay;
-        const maxDeliveryAttempts = ((_e = config.retries) !== null && _e !== void 0 ? _e : 0) + 1;
+        const maxDeliveryAttempts = ((_f = config.retries) !== null && _f !== void 0 ? _f : 0) + 1;
         const enableDeadLetter = maxDeliveryAttempts > 0;
         if (enableDeadLetter) {
             yield ensureSubscription(client, Object.assign(Object.assign({}, config), { retries: exports.RETRY_FOREVER, name: dlTopicName, topicName: dlTopicName, deadLetterTopicName: undefined, subscriptionName: `${dlTopicName}.sub`, retryDelay: minRetryDelay, maxRetryDelay }));

package/dist/gcppubsub/connection.js

@@ -10,16 +10,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GcpPubSubConnection = exports.GCP_PUBSUB_TRANSPORT = void 0;
-const lazy_load_js_1 = require("../lazy-load.js");
+const gcppubsub_helper_js_1 = require("./gcppubsub-helper.js");
 exports.GCP_PUBSUB_TRANSPORT = "gcp-pubsub";
 class GcpPubSubConnection {
     static create(config, logger) {
         return __awaiter(this, void 0, void 0, function* () {
-            const GcpPubSubPackage = yield (0, lazy_load_js_1.lazyLoad)({
-                packageName: "@google-cloud/pubsub",
-                context: "GcpPubSubConnection",
-            });
-            const client = new GcpPubSubPackage.PubSub(Object.assign({ projectId: config.projectId }, config.clientConfig));
+            const gcppubsubPackage = yield (0, gcppubsub_helper_js_1.lazyLoadGcpPubSubPackage)("GcpPubSubConnection");
+            const client = new gcppubsubPackage.PubSub(Object.assign({ projectId: config.projectId }, config.clientConfig));
             return new GcpPubSubConnection(client, logger);
         });
     }

package/dist/gcppubsub/gcppubsub-helper.d.ts

@@ -0,0 +1,8 @@
+import type * as gcppubsub from "@google-cloud/pubsub";
+export declare const GCP_PUBSUB_TRANSPORT = "gcp-pubsub";
+export interface GcpPubSubConnectionConfig {
+    projectId?: string;
+    clientConfig?: gcppubsub.ClientConfig;
+}
+export declare function getGcpPubSubPackage(): typeof gcppubsub | undefined;
+export declare function lazyLoadGcpPubSubPackage(context: string): Promise<typeof gcppubsub>;

package/dist/gcppubsub/gcppubsub-helper.js

@@ -0,0 +1,29 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GCP_PUBSUB_TRANSPORT = void 0;
+exports.getGcpPubSubPackage = getGcpPubSubPackage;
+exports.lazyLoadGcpPubSubPackage = lazyLoadGcpPubSubPackage;
+const lazy_load_js_1 = require("../lazy-load.js");
+exports.GCP_PUBSUB_TRANSPORT = "gcp-pubsub";
+let GcpPubSubPackageInstance = undefined;
+function getGcpPubSubPackage() {
+    return GcpPubSubPackageInstance;
+}
+function lazyLoadGcpPubSubPackage(context) {
+    return __awaiter(this, void 0, void 0, function* () {
+        GcpPubSubPackageInstance = yield (0, lazy_load_js_1.lazyLoad)({
+            packageName: "@google-cloud/pubsub",
+            context,
+        });
+        return GcpPubSubPackageInstance;
+    });
+}

package/dist/gcppubsub/subscriber.js

@@ -10,14 +10,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GcpPubSubSubscriber = void 0;
-const pubsub_1 = require("@google-cloud/pubsub");
 const trace_1 = require("@palmetto/trace");
 const interfaces_js_1 = require("../interfaces.js");
 const errors_js_1 = require("../errors.js");
 const create_log_error_payload_js_1 = require("../create-log-error-payload.js");
 const config_js_1 = require("./config.js");
 const connection_js_1 = require("./connection.js");
-const pubsub_2 = require("@google-cloud/pubsub");
+const gcppubsub_helper_js_1 = require("./gcppubsub-helper.js");
 const SHUTDOWN_DELAY = 1000;
 class GcpPubSubSubscriber {
     constructor(connection, logger) {
@@ -29,7 +28,7 @@ class GcpPubSubSubscriber {
     }
     startSubscribe(config, onMessage) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d;
+            var _a, _b, _c, _d, _e;
             const subscriptionName = (0, config_js_1.getSubscriptionName)(config);
             if (this.subscriptions.has(subscriptionName)) {
                 throw new errors_js_1.AlreadySubscribingError(subscriptionName);
@@ -51,9 +50,9 @@ class GcpPubSubSubscriber {
             const subscription = this.connection.client.subscription(subscriptionName, Object.assign({ flowControl: {
                     maxMessages: (_c = config.maxMessages) !== null && _c !== void 0 ? _c : 5,
                 }, closeOptions: {
-                    behavior: pubsub_2.SubscriptionCloseBehaviors.WaitForProcessing,
-                    timeout: pubsub_1.Duration.from({
-                        milliseconds: (_d = config.shutdownDelay) !== null && _d !== void 0 ? _d : SHUTDOWN_DELAY,
+                    behavior: "WAIT",
+                    timeout: (_d = (0, gcppubsub_helper_js_1.getGcpPubSubPackage)()) === null || _d === void 0 ? void 0 : _d.Duration.from({
+                        milliseconds: (_e = config.shutdownDelay) !== null && _e !== void 0 ? _e : SHUTDOWN_DELAY,
                     }),
                 } }, config.subscriptionOptions));
             const messageHandler = (message) => {

package/dist/interfaces.d.ts

@@ -1,22 +1,28 @@
 import { z } from "zod";
+/**
+ * The schema for the metadata of a message. This is added by the publisher if not provided, but can also be provided by the publisher if they want to set specific values.
+ */
 export declare const MetaSchema: z.ZodObject<{
-    createdAt: z.ZodISODateTime;
-    schemaId: z.ZodString;
-    publishedBy: z.ZodString;
+    createdAt: z.ZodOptional<z.ZodISODateTime>;
+    schemaId: z.ZodOptional<z.ZodString>;
+    publishedBy: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
+/**
+ * The id and meta fields that are added to all messages by the publisher if not provided.
+ */
 export declare const IdMetaSchema: z.ZodObject<{
     id: z.ZodOptional<z.ZodString>;
     meta: z.ZodOptional<z.ZodObject<{
-        createdAt: z.ZodISODateTime;
-        schemaId: z.ZodString;
-        publishedBy: z.ZodString;
+        createdAt: z.ZodOptional<z.ZodISODateTime>;
+        schemaId: z.ZodOptional<z.ZodString>;
+        publishedBy: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>>;
 }, z.core.$strip>;
 export type Meta = z.infer<typeof MetaSchema>;
 /**
  * All messages should be based on this schema
  */
-export type BaseMessage = z.infer<typeof IdMetaSchema>;
+export type BaseMessage = Record<string, unknown> & z.infer<typeof IdMetaSchema>;
 export interface PublishMessageOptions {
     /**
      * The date and time at which the message should be published. Only supported by BullMq at this time.

package/dist/interfaces.js

@@ -2,13 +2,35 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MessageResult = exports.IdMetaSchema = exports.MetaSchema = void 0;
 const zod_1 = require("zod");
+/**
+ * The schema for the metadata of a message. This is added by the publisher if not provided, but can also be provided by the publisher if they want to set specific values.
+ */
 exports.MetaSchema = zod_1.z.object({
-    createdAt: zod_1.z.iso.datetime(),
-    schemaId: zod_1.z.string(),
-    publishedBy: zod_1.z.string(),
+    /**
+     * The date and time at which the message was created.
+     */
+    createdAt: zod_1.z.iso.datetime().optional(),
+    /**
+     * The unique identifier for the schema used by the message. Currently a hash of the JSON schema definition, but could be changed in the future to for example a UUID or a URL where the schema can be found.
+     */
+    schemaId: zod_1.z.string().optional(),
+    /**
+     * The identifier of the entity that published the message. (currently an empty string, but could be used to identify the service that published the message)
+     */
+    publishedBy: zod_1.z.string().optional(),
 });
+/**
+ * The id and meta fields that are added to all messages by the publisher if not provided.
+ */
 exports.IdMetaSchema = zod_1.z.object({
+    /**
+     * A unique identifier for the message. If not provided, one will be generated during publish.
+     */
     id: zod_1.z.string().optional(),
+    /**
+     * Metadata about the message, such as when it was created, which schema it uses, etc.
+     * This is optional when publishing a message, but will be added by the publisher if not provided.
+     */
     meta: exports.MetaSchema.optional(),
 });
 var MessageResult;

package/dist/publisher.js

@@ -80,6 +80,7 @@ class Publisher {
                 }
                 else {
                     msg.meta.schemaId = schemaId;
+                    msg.meta.createdAt = new Date().toISOString();
                 }
                 const check = schema.safeEncode(msg);
                 if (!check.success) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@palmetto/pubsub",
-  "version": "3.2.2",
+  "version": "3.2.4",
   "repository": {
     "type": "git",
     "url": "https://github.com/palmetto/galaxy"