@hiiretail/gcp-infra-cli 0.74.0 → 0.75.0

package/generators/resources/pubsub/append.js

@@ -1,7 +1,15 @@
 const yaml = require('js-yaml');
 const fs = require('fs');
 
-const appendNotIncludePull = async (inputs, subscriptionFilePath) => {
+const addDLQ = async (yamlArray, env, dlqTopic) => {
+  if (env === 'prod') {
+    /* eslint-disable */
+    yamlArray[0].dead_letter_topic = dlqTopic;
+    /* eslint-enable */
+  }
+};
+
+const appendNotIncludePull = async (inputs, subscriptionFilePath, dlqTopic) => {
   const pullArray = [];
 
   pullArray.push(
@@ -12,11 +20,17 @@ const appendNotIncludePull = async (inputs, subscriptionFilePath) => {
     },
   );
 
+  await addDLQ(pullArray, inputs.env, dlqTopic);
   const yamlPullArray = yaml.dump(pullArray);
   fs.appendFileSync(subscriptionFilePath, `pull_subscriptions:\n${yamlPullArray}`);
 };
 
-const appendIncludePull = async (fileContent, originalContentYaml, subscriptionFilePath, input) => {
+const appendIncludePull = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic) => {
   if (fileContent.includes('pull_subscriptions')) {
     const pullArray = Object.values(originalContentYaml.pull_subscriptions);
     const yamlPullArray = yaml.dump(pullArray);
@@ -33,6 +47,7 @@ const appendIncludePull = async (fileContent, originalContentYaml, subscriptionF
     },
   );
 
+  await addDLQ(newPullArray, input.env, dlqTopic);
   const finalYamlPullArray = yaml.dump(newPullArray);
   fs.appendFileSync(subscriptionFilePath, finalYamlPullArray);
 
@@ -43,7 +58,7 @@ const appendIncludePull = async (fileContent, originalContentYaml, subscriptionF
   }
 };
 
-const appendNotIncludePush = async (inputs, subscriptionFilePath) => {
+const appendNotIncludePush = async (inputs, subscriptionFilePath, dlqTopic) => {
   const pushArray = [];
 
   pushArray.push(
@@ -56,11 +71,17 @@ const appendNotIncludePush = async (inputs, subscriptionFilePath) => {
     },
   );
 
+  await addDLQ(pushArray, inputs.env, dlqTopic);
   const yamlPushArray = yaml.dump(pushArray);
   fs.appendFileSync(subscriptionFilePath, `push_subscriptions:\n${yamlPushArray}`);
 };
 
-const appendIncludePush = async (fileContent, originalContentYaml, subscriptionFilePath, input) => {
+const appendIncludePush = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic) => {
   if (fileContent.includes('push_subscriptions')) {
     const pushArray = Object.values(originalContentYaml.push_subscriptions);
     const yamlPushArray = yaml.dump(pushArray);
@@ -78,6 +99,7 @@ const appendIncludePush = async (fileContent, originalContentYaml, subscriptionF
       expiration_policy: '',
     },
   );
+  await addDLQ(newPushArray, input.env, dlqTopic);
   const yamlPushArray = yaml.dump(newPushArray);
   fs.appendFileSync(subscriptionFilePath, yamlPushArray);
 

package/generators/resources/pubsub/handle-subscribers.js

@@ -4,7 +4,13 @@ const {
   appendNotIncludePull, appendIncludePull, appendNotIncludePush, appendIncludePush,
 } = require('./append');
 
-const handleSubscribers = async (env, answers, oidcEmail, pushEndpoint, subscriptionFilePath) => {
+const handleSubscribers = async (
+  env,
+  answers,
+  oidcEmail,
+  pushEndpoint,
+  subscriptionFilePath,
+  dlqTopic) => {
   const {
     subscriberName,
     existingTopic,
@@ -22,22 +28,35 @@ const handleSubscribers = async (env, answers, oidcEmail, pushEndpoint, subscrip
     audience,
     oidcEmail,
     pushEndpoint,
+    dlqTopic,
   };
 
   if (pushOrPull === 'pull') {
     if (subscriptionFileContent.length === 0 || !subscriptionFileContent.includes('pull_subscriptions')) {
-      await appendNotIncludePull(inputs, subscriptionFilePath);
+      await appendNotIncludePull(inputs, subscriptionFilePath, dlqTopic);
     } else {
       const originalContentYaml = yaml.load(subscriptionFileContent);
       const fileContent = subscriptionFileContent;
-      await appendIncludePull(fileContent, originalContentYaml, subscriptionFilePath, inputs);
+      await appendIncludePull(
+        fileContent,
+        originalContentYaml,
+        subscriptionFilePath,
+        inputs,
+        dlqTopic,
+      );
     }
   } else if (subscriptionFileContent.length === 0 || !subscriptionFileContent.includes('push_subscriptions')) {
-    await appendNotIncludePush(inputs, subscriptionFilePath);
+    await appendNotIncludePush(inputs, subscriptionFilePath, dlqTopic);
   } else {
     const originalContentYaml = yaml.load(subscriptionFileContent);
     const fileContent = subscriptionFileContent;
-    await appendIncludePush(fileContent, originalContentYaml, subscriptionFilePath, inputs);
+    await appendIncludePush(
+      fileContent,
+      originalContentYaml,
+      subscriptionFilePath,
+      inputs,
+      dlqTopic,
+    );
   }
 };
 

package/generators/resources/pubsub/index.js

@@ -170,6 +170,18 @@ module.exports = class extends BaseGenerator {
       externalSub,
     } = this.answers;
 
+    const dlqTopicName = `${getProjectId('prod').split('-')[0]}-common-dlq`;
+    let dlqTopic = `projects/${getProjectId('prod')}/topics/${dlqTopicName}`;
+
+    const dlqTopicDirPath = path.join(process.cwd(), 'infra', 'prod', 'pubsub', dlqTopicName);
+    this.fs.copyTpl(
+      this.templatePath('pubsub-dlq/terragrunt.hcl'),
+      this.destinationPath(`${dlqTopicDirPath}/terragrunt.hcl`),
+      {
+        ...this.answers,
+      },
+    );
+
     ['prod', 'staging'].forEach(async (env) => {
       if (createResource === 'topic') {
         const topicDirPath = path.join(process.cwd(), 'infra', env, 'pubsub', topicName);
@@ -182,6 +194,7 @@ module.exports = class extends BaseGenerator {
               env,
               topicName,
               subscriberName,
+              dlqTopic,
             },
           );
         });
@@ -212,6 +225,7 @@ module.exports = class extends BaseGenerator {
               audience,
               oidcEmail,
               pushEndpoint,
+              dlqTopic,
             },
           );
         }
@@ -228,7 +242,7 @@ module.exports = class extends BaseGenerator {
           );
         });
 
-        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, `${subscriptionDirPath}/subscribers.yaml`);
+        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, `${subscriptionDirPath}/subscribers.yaml`, dlqTopic);
       }
       if (createResource === 'subscription' && externalSub === 'yes') {
         const externalDirPath = path.join(process.cwd(), 'infra', env, 'pubsub', existingTopic, clanName);
@@ -259,6 +273,7 @@ module.exports = class extends BaseGenerator {
           oidcEmail = `${oidcName}@${stagingProjectIdConsumer}.iam.gserviceaccount.com`;
         } else {
           oidcEmail = `${oidcName}@${prodProjectIdConsumer}.iam.gserviceaccount.com`;
+          dlqTopic = `projects/${prodProjectIdConsumer}/topics/${prodProjectIdConsumer.split('-')[0]}-common-dlq`;
         }
         if (!fs.existsSync(externalSubPath)) {
           fs.writeFileSync(externalSubPath, '');
@@ -273,11 +288,19 @@ module.exports = class extends BaseGenerator {
               audience,
               oidcEmail,
               pushEndpoint,
+              dlqTopic,
             },
           );
         }
 
-        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, externalSubPath);
+        await handleSubscribers(
+          env,
+          this.answers,
+          oidcEmail,
+          pushEndpoint,
+          externalSubPath,
+          dlqTopic,
+        );
       }
     });
 

package/generators/resources/pubsub/templates/pubsub/subscribers.yaml

@@ -1,9 +1,11 @@
 <% if (createResource == 'topic') { %><% } %><% if (createResource == 'subscription' && pushOrPull == 'push') { %>push_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  push_endpoint: "<%-pushEndpoint%>"
+  push_endpoint: "<%-pushEndpoint%>" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   oidc_service_account_email: "<%-oidcEmail%>"
   audience: "<%-audience%>"
   expiration_policy: ""<% } %><% if (createResource == 'subscription' && pushOrPull == 'pull') { %>pull_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  ack_deadline_seconds: "60"
+  ack_deadline_seconds: "60" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   expiration_policy: ""<% } %>

package/generators/resources/pubsub/templates/pubsub-dlq/terragrunt.hcl

@@ -0,0 +1,41 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/terraform-google-modules/terraform-google-pubsub//?ref=v3.0.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+  common_vars  = read_terragrunt_config(find_in_parent_folders("common.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge (
+    local.project_vars.locals,
+    {
+      topic                = "${local.common_vars.locals.clan_name}-common-dlq"
+      create_subscriptions = true
+      create_topic         = true
+      push_subscriptions = [
+          {
+          name                        = "dlq-message-handler-subscription",
+          push_endpoint               = "https://europe-west1-sre-prod-5462.cloudfunctions.net/dlq-message-handler",
+          audience                    = "https://europe-west1-sre-prod-5462.cloudfunctions.net/dlq-message-handler",
+          expiration_policy           = "",
+          oidc_service_account_email  = "pubsub-dlq-handler@${local.project_vars.locals.project_id}.iam.gserviceaccount.com",
+          },
+        ],
+      topic_labels = {
+        cc = local.common_vars.locals.cost_center
+      }
+      subscription_labels = {
+        cc = local.common_vars.locals.cost_center
+      }
+      grant_token_creator = false,
+    }
+)

package/generators/resources/pubsub/templates/pubsub-external/subscribers.yaml

@@ -1,9 +1,11 @@
 <% if (pushOrPull == 'push') { %>push_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  push_endpoint: "<%-pushEndpoint%>"
+  push_endpoint: "<%-pushEndpoint%>" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   oidc_service_account_email: "<%-oidcEmail%>"
   audience: "<%-audience%>"
   expiration_policy: ""<% } %><% if (pushOrPull == 'pull') { %>pull_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  ack_deadline_seconds: "60"
+  ack_deadline_seconds: "60" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   expiration_policy: ""<% } %>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.74.0",
+  "version": "0.75.0",
   "description": "Infrastructure as code generator for GCP.",
   "main": "src/cli.js",
   "bin": {