@shushed/helpers 0.0.36 → 0.0.38

package/dist/index.d.ts

@@ -28743,6 +28743,7 @@ declare class EnvEngine extends Runtime {
 
 type Message = {
     buildshipId: string;
+    exportableToBigQuery: boolean;
     body: any;
     sourceSystem: string;
     targetSystem: string;
@@ -28750,6 +28751,7 @@ type Message = {
     recordModifiedAt: Date | null;
     originalPublishTime: Date | null;
     publishTime: Date | null;
+    extraAttributes: Record<string, string>;
 };
 type LastModifiedKeys = 'last_modified_date_time' | 'last_modified_date' | 'last_modified_datetime' | 'record_modified_at' | 'lastModifiedDateTime' | 'lastModifiedAt' | 'last_modified_at' | 'SystemModifiedAt' | 'system_modified_at' | 'lastUpdatedAt' | 'modified_at' | 'last_updated_at' | 'modifiedAt';
 type AtLeastOne<T, Keys extends keyof T = keyof T> = Partial<T> & {
@@ -28792,6 +28794,7 @@ declare class PubSubHelper extends Runtime {
     private findTopic;
     private ensureTopicExists;
     private getNameFromFullyQualifiedName;
+    private updateSubscription;
     private ensureSubscribtionExists;
     private ensureDLQSubscriptionExists;
     deleteSubscription(subscriptionURN: string): Promise<any>;

package/dist/index.js

@@ -678,8 +678,13 @@ var env_default = EnvEngine;
 
 // src-public/pubsub.ts
 var import_co_body = __toESM(require("co-body"));
+var import_lodash = __toESM(require("lodash.isequal"));
+var import_lodash2 = __toESM(require("lodash.omit"));
+function isSubscriptionOptionsPull(opts) {
+  return "endpointUrl" in opts && opts.endpoinrtUrl === null;
+}
 function isSubscriptionOptionsPush(opts) {
-  return "endpointUrl" in opts;
+  return "endpointUrl" in opts && opts.endpoinrtUrl !== null;
 }
 var PubSubHelper = class extends Runtime {
   constructor(opts, pubSub) {
@@ -705,7 +710,9 @@ var PubSubHelper = class extends Runtime {
       originalPublishTime: parseDateOrDefault(rawMessage.attributes?.original_publish_time, parseDateOrDefault(rawMessage.publish_time)),
       buildshipId: rawMessage.attributes?.buildship_id || "unknown",
       publishTime: parseDateOrDefault(rawMessage.publish_time),
-      body: messageBody
+      body: messageBody,
+      exportableToBigQuery: !!rawMessage.attributes?.bigquery,
+      extraAttributes: (0, import_lodash2.default)(rawMessage.attributes || {}, ["source_system", "target_system", "process_started_at", "record_modified_at", "original_publish_time", "buildship_id", "publish_time", "bigquery"])
     };
     return message;
   }
@@ -726,16 +733,21 @@ var PubSubHelper = class extends Runtime {
     for (let i = 0; i < normPayloads.length; i++) {
       const normPayload = normPayloads[i];
       promises.push((async () => {
+        const attributes = {
+          ...(0, import_lodash2.default)(options.extraAttributes || {}, ["source_system", "target_system", "process_started_at", "record_modified_at", "original_publish_time", "buildship_id", "publish_time", "bigquery"]),
+          source_system: options.sourceSystem,
+          target_system: options.targetSystem || orgMessage?.targetSystem || "*",
+          process_started_at: (options.processStartedAt || orgMessage?.processStartedAt || /* @__PURE__ */ new Date()).toISOString(),
+          record_modified_at: (getEventTime(normPayload, null) || /* @__PURE__ */ new Date()).toISOString(),
+          original_publish_time: (orgMessage?.originalPublishTime || orgMessage?.publishTime || /* @__PURE__ */ new Date())?.toDateString() || "",
+          buildship_id: this.systemEnvName + "-" + this.workflowId + "-" + this.triggerId
+        };
+        if (options.exportableToBigQuery) {
+          attributes.bigquery = "yes";
+        }
         const result = await topic.publishMessage({
           data: Buffer.from(JSON.stringify(normPayload)),
-          attributes: {
-            source_system: options.sourceSystem,
-            target_system: options.targetSystem || orgMessage?.targetSystem || "*",
-            process_started_at: (options.processStartedAt || orgMessage?.processStartedAt || /* @__PURE__ */ new Date()).toISOString(),
-            record_modified_at: (getEventTime(normPayload, null) || /* @__PURE__ */ new Date()).toISOString(),
-            original_publish_time: (orgMessage?.originalPublishTime || orgMessage?.publishTime || /* @__PURE__ */ new Date())?.toDateString() || "",
-            buildship_id: this.systemEnvName + "-" + this.workflowId + "-" + this.triggerId
-          }
+          attributes
         });
         return result;
       })());
@@ -760,12 +772,20 @@ var PubSubHelper = class extends Runtime {
     let bqSubscription = null;
     if (opts.bigQueryTableId) {
       const bigQuerySubscriptionName = `${subscriptionName}-bq`;
+      const bigQueryDlqTopicName = `${opts.topicName}-bq-dlq`;
+      const bigQueryDlqSubscriptionName = `${subscriptionName}-bq-dlq`;
+      dlqTopic = await this.ensureTopicExists(bigQueryDlqTopicName);
+      dlqSubscription = await this.ensureDLQSubscriptionExists(bigQueryDlqSubscriptionName, {
+        topic: dlqTopic
+      });
       bqSubscription = await this.ensureSubscribtionExists(bigQuerySubscriptionName, {
         topic,
-        tableId: opts.bigQueryTableId
+        tableId: opts.bigQueryTableId,
+        dlqTopic
       });
     } else {
       await this.deleteSubscription(`${subscriptionName}-bq`);
+      await this.deleteSubscription(`${subscriptionName}-bq-dlq`);
     }
     const subscription = await this.ensureSubscribtionExists(subscriptionName, {
       topic,
@@ -790,8 +810,46 @@ var PubSubHelper = class extends Runtime {
   }
   async ensureTopicExists(topicName) {
     let topic = await this.findTopic(topicName);
-    if (!topic) {
-      await this.pubSub.createTopic(topicName);
+    const metadata = {
+      labels: {
+        "created-at": (/* @__PURE__ */ new Date()).toISOString(),
+        "last-modified-at": (/* @__PURE__ */ new Date()).toISOString(),
+        "created-by-bs-triggerid": this.triggerId,
+        "created-by-bs-workflowid": this.workflowId,
+        "bs-envname": this.systemEnvName,
+        "env": this.envName,
+        "used-by": ""
+      }
+    };
+    const [topicMetadata] = await topic?.getMetadata() || [null];
+    const currentUsedBy = (topicMetadata?.labels?.["used-by"] || "").split(",").map((x) => x.split(":").slice(0, 2)).reduce((acc, [wid, envName]) => {
+      acc[wid] = acc[wid] || [];
+      acc[wid].push(`${envName}---${this.triggerId}`);
+      return acc;
+    }, {});
+    if (currentUsedBy[this.workflowId] && currentUsedBy[this.workflowId].indexOf(`${this.systemEnvName}---${this.triggerId}`) !== -1) {
+      currentUsedBy[this.workflowId].push(`${this.systemEnvName}---${this.triggerId}`);
+    }
+    if (topic) {
+      const currentUsedByString = Object.entries(currentUsedBy).reduce((acc, [wid, x]) => {
+        x.forEach((y) => {
+          acc.push(wid + ":" + y);
+        });
+        return acc;
+      }, []).join(",");
+      metadata.labels["used-by"] = currentUsedByString;
+      const simMetadata = Object.assign({}, topicMetadata?.labels || {}, metadata.labels);
+      if (!(0, import_lodash.default)(topicMetadata?.labels, simMetadata)) {
+        simMetadata["last-modified-at"] = (/* @__PURE__ */ new Date()).toISOString();
+        await topic.setMetadata({
+          labels: simMetadata
+        });
+      }
+    } else {
+      await this.pubSub.createTopic({
+        name: topicName,
+        labels: metadata.labels
+      });
       topic = await this.findTopic(topicName);
     }
     if (!topic) {
@@ -803,66 +861,119 @@ var PubSubHelper = class extends Runtime {
     const name = fullyQName.split("/");
     return name[name.length - 1];
   }
-  async ensureSubscribtionExists(subscriptionName, opts) {
-    let [subscriptions] = await opts.topic.getSubscriptions();
-    let subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName);
-    let endpointHasChanged = false;
-    let retryLimitHasChanged = false;
-    let ackDeadlineHasChanged = false;
-    let retryDelayHasChanged = false;
-    let shouldRecreate = false;
-    let dlqHasChanged = false;
-    let tableIdHasChanged = false;
-    if (subscription) {
-      const [subscriptionMetaData] = await subscription.getMetadata();
-      if (!isSubscriptionOptionsPush(opts)) {
-        tableIdHasChanged = (subscriptionMetaData.bigqueryConfig?.table || null) !== (opts.tableId || null);
-      } else {
-        endpointHasChanged = subscriptionMetaData.pushConfig?.pushEndpoint != opts.endpointUrl;
-        dlqHasChanged = !subscriptionMetaData.deadLetterPolicy && !!opts.dlqTopic || !!subscriptionMetaData.deadLetterPolicy?.deadLetterTopic && !opts.dlqTopic;
-        retryDelayHasChanged = subscriptionMetaData.retryPolicy?.minimumBackoff?.seconds != opts.retryMinDelay;
-        retryLimitHasChanged = subscriptionMetaData.deadLetterPolicy?.maxDeliveryAttempts != opts.retryLimit;
-        ackDeadlineHasChanged = subscriptionMetaData.ackDeadlineSeconds !== (opts.ackDeadline || 90);
+  async updateSubscription(subscriptionName, subscription, subscriptionMetaData, inputOpts) {
+    const defaultOptions = {
+      retryLimit: 5,
+      retryMinDelay: 10,
+      retryMaxDelay: 360,
+      ackDeadline: 90
+    };
+    const opts = Object.assign({}, defaultOptions, inputOpts);
+    let nextMetadata = null;
+    if (isSubscriptionOptionsPush(opts)) {
+      if (opts.endpointUrl !== subscriptionMetaData?.pushConfig?.pushEndpoint || this.serviceAccountEmail !== subscriptionMetaData?.pushConfig.oidcToken?.serviceAccountEmail || process.env.GCLOUD_PROJECT + "/" + this.triggerId !== subscriptionMetaData.pushConfig.oidcToken.audience) {
+        if (subscriptionMetaData?.pushConfig?.pushEndpoint && subscription) {
+          await subscription.modifyPushConfig({
+            pushEndpoint: opts.endpointUrl,
+            oidcToken: {
+              serviceAccountEmail: this.serviceAccountEmail,
+              audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId
+            }
+          });
+        } else {
+          nextMetadata = nextMetadata || {};
+          nextMetadata.pushConfig = {
+            pushEndpoint: opts.endpointUrl,
+            oidcToken: {
+              serviceAccountEmail: this.serviceAccountEmail,
+              audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId
+            }
+          };
+        }
       }
-      shouldRecreate = endpointHasChanged || retryDelayHasChanged || retryLimitHasChanged || dlqHasChanged || tableIdHasChanged || ackDeadlineHasChanged;
-    }
-    if (subscription && shouldRecreate) {
-      await subscription.delete();
-    }
-    const payload = {};
-    if (!isSubscriptionOptionsPush(opts)) {
-      payload.ackDeadlineSeconds = 60;
-      payload.bigqueryConfig = {
-        table: opts.tableId,
-        writeMetadata: true,
-        dropUnknownFields: true,
-        serviceAccountEmail: this.serviceAccountEmail,
-        useTableSchema: false
-      };
     } else {
-      payload.ackDeadlineSeconds = opts.ackDeadline || 90;
-      payload.pushConfig = {
-        pushEndpoint: opts.endpointUrl,
-        oidcToken: {
-          serviceAccountEmail: this.serviceAccountEmail,
-          audience: process.env.GCLOUD_PROJECT + "/" + this.triggerId
-        }
+      nextMetadata = nextMetadata || {};
+      nextMetadata.pushConfig = null;
+    }
+    if (opts.dlqTopic) {
+      if (opts.dlqTopic.name !== subscriptionMetaData?.deadLetterPolicy?.deadLetterTopic || opts.retryLimit !== subscriptionMetaData?.deadLetterPolicy.maxDeliveryAttempts) {
+        nextMetadata = nextMetadata || {};
+        nextMetadata.deadLetterPolicy = {
+          deadLetterTopic: opts.dlqTopic.name,
+          maxDeliveryAttempts: opts.retryLimit
+        };
+      }
+    } else if (subscriptionMetaData?.deadLetterPolicy?.deadLetterTopic) {
+      nextMetadata = nextMetadata || {
+        deadLetterPolicy: null
       };
-      payload.retryPolicy = {
+    }
+    if (opts.retryMinDelay !== subscriptionMetaData?.retryPolicy?.minimumBackoff?.seconds || opts.retryMaxDelay !== subscriptionMetaData?.retryPolicy?.maximumBackoff?.seconds) {
+      nextMetadata = nextMetadata || {};
+      nextMetadata.retryPolicy = {
         minimumBackoff: { seconds: opts.retryMinDelay },
         maximumBackoff: { seconds: opts.retryMaxDelay }
       };
-      if (opts.dlqTopic) {
-        payload.deadLetterPolicy = {
-          deadLetterTopic: this.getNameFromFullyQualifiedName(opts.dlqTopic.name),
-          maxDeliveryAttempts: opts.retryLimit
-        };
+    }
+    const proposedLabels = {
+      "created-at": subscriptionMetaData?.labels?.["created-at"] || (/* @__PURE__ */ new Date()).toISOString(),
+      "last-modified-at": subscriptionMetaData?.labels?.["created-at"] || (/* @__PURE__ */ new Date()).toISOString(),
+      "env": this.envName,
+      "bs-triggerid": this.triggerId,
+      "bs-workflowid": this.workflowId,
+      "bs-envname": this.systemEnvName,
+      "bs-runtimeurl": this.runtimeUrl,
+      "bs-is-deadletter": subscriptionName.indexOf("-dlq") > -1 ? "true" : "false",
+      "bs-is-push-enabled": isSubscriptionOptionsPush(opts) && opts.endpointUrl ? "true" : "false",
+      "bs-bigquery-tableid": !(isSubscriptionOptionsPush(opts) || isSubscriptionOptionsPull(opts)) && opts.tableId ? opts.tableId : "",
+      "bs-is-bigquery": !(isSubscriptionOptionsPush(opts) || isSubscriptionOptionsPull(opts)) && opts.tableId ? "true" : "false"
+    };
+    const simLabels = Object.assign({}, subscriptionMetaData?.labels || {}, proposedLabels);
+    if (!(0, import_lodash.default)(simLabels, subscriptionMetaData?.labels)) {
+      simLabels["last-modified-at"] = (/* @__PURE__ */ new Date()).toISOString();
+      nextMetadata = nextMetadata || {};
+      nextMetadata.labels = simLabels;
+    }
+    if (opts.ackDeadline !== subscriptionMetaData?.ackDeadlineSeconds) {
+      nextMetadata = nextMetadata || {};
+      nextMetadata.ackDeadlineSeconds = opts.ackDeadline;
+    }
+    if (!isSubscriptionOptionsPull(opts) && !isSubscriptionOptionsPush(opts)) {
+      if (opts.tableId !== subscriptionMetaData?.bigqueryConfig?.table || this.serviceAccountEmail !== subscriptionMetaData.bigqueryConfig?.serviceAccountEmail) {
+        if (opts.tableId) {
+          nextMetadata = nextMetadata || {};
+          nextMetadata.bigqueryConfig = {
+            table: opts.tableId,
+            writeMetadata: true,
+            dropUnknownFields: true,
+            serviceAccountEmail: this.serviceAccountEmail,
+            useTableSchema: false
+          };
+          nextMetadata.filter = `attributes:bigquery`;
+        } else {
+          nextMetadata = nextMetadata || {};
+          nextMetadata.bigqueryConfig = null;
+        }
       }
+    } else if (subscriptionMetaData?.bigqueryConfig?.table) {
+      nextMetadata = nextMetadata || {};
+      nextMetadata.bigqueryConfig = null;
     }
-    if (!subscription || shouldRecreate) {
-      await opts.topic.createSubscription(subscriptionName, payload);
+    if (subscription && nextMetadata) {
+      await subscription.setMetadata(nextMetadata);
+      return null;
+    }
+    return nextMetadata;
+  }
+  async ensureSubscribtionExists(subscriptionName, opts) {
+    let [subscriptions] = await opts.topic.getSubscriptions();
+    let subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName) || null;
+    const [subscriptionMetaData] = await subscription?.getMetadata() || [null];
+    const createPayload = await this.updateSubscription(subscriptionName, subscription, subscriptionMetaData, opts);
+    if (createPayload) {
+      await opts.topic.createSubscription(subscriptionName, createPayload);
       [subscriptions] = await opts.topic.getSubscriptions();
-      subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName);
+      subscription = subscriptions.find((subscription2) => this.getNameFromFullyQualifiedName(subscription2.name) === subscriptionName) || null;
     }
     if (!subscription) {
       throw new Error(`Invariant: While creating subscription - subscription is not found`);

package/dist/package.json

@@ -10,7 +10,10 @@
     "ajv-formats": "^3.0.1",
     "co-body": "^6.2.0",
     "jose": "^6.0.11",
-    "lodash.clonedeep": "^4.5.0"
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.isequal": "^4.5.0",
+    "lodash.omit": "^4.5.0",
+    "lodash.pick": "^4.4.0"
   },
   "files": [
     "dist"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shushed/helpers",
-  "version": "0.0.36",
+  "version": "0.0.38",
   "author": "",
   "license": "UNLICENSED",
   "description": "",
@@ -10,7 +10,10 @@
     "ajv-formats": "^3.0.1",
     "co-body": "^6.2.0",
     "jose": "^6.0.11",
-    "lodash.clonedeep": "^4.5.0"
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.isequal": "^4.5.0",
+    "lodash.omit": "^4.5.0",
+    "lodash.pick": "^4.4.0"
   },
   "files": [
     "dist"