@hiiretail/gcp-infra-cli 0.74.0 → 0.75.2

package/README.md

@@ -39,7 +39,7 @@ Changelog of key features is available in [CHANGELOG.md](./CHANGELOG.md)
 
 Development tools needed are:
 
-  * Latest Node 12 LTS release
+  * Latest Node 16 LTS release
   * Docker
   * Pre-commit
 

package/generators/resources/pubsub/append.js

@@ -1,7 +1,15 @@
 const yaml = require('js-yaml');
 const fs = require('fs');
 
-const appendNotIncludePull = async (inputs, subscriptionFilePath) => {
+const addDLQ = async (yamlArray, env, dlqTopic) => {
+  if (env === 'prod') {
+    /* eslint-disable */
+    yamlArray[0].dead_letter_topic = dlqTopic;
+    /* eslint-enable */
+  }
+};
+
+const appendNotIncludePull = async (inputs, subscriptionFilePath, dlqTopic) => {
   const pullArray = [];
 
   pullArray.push(
@@ -12,11 +20,17 @@ const appendNotIncludePull = async (inputs, subscriptionFilePath) => {
     },
   );
 
+  await addDLQ(pullArray, inputs.env, dlqTopic);
   const yamlPullArray = yaml.dump(pullArray);
   fs.appendFileSync(subscriptionFilePath, `pull_subscriptions:\n${yamlPullArray}`);
 };
 
-const appendIncludePull = async (fileContent, originalContentYaml, subscriptionFilePath, input) => {
+const appendIncludePull = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic) => {
   if (fileContent.includes('pull_subscriptions')) {
     const pullArray = Object.values(originalContentYaml.pull_subscriptions);
     const yamlPullArray = yaml.dump(pullArray);
@@ -33,6 +47,7 @@ const appendIncludePull = async (fileContent, originalContentYaml, subscriptionF
     },
   );
 
+  await addDLQ(newPullArray, input.env, dlqTopic);
   const finalYamlPullArray = yaml.dump(newPullArray);
   fs.appendFileSync(subscriptionFilePath, finalYamlPullArray);
 
@@ -43,7 +58,7 @@ const appendIncludePull = async (fileContent, originalContentYaml, subscriptionF
   }
 };
 
-const appendNotIncludePush = async (inputs, subscriptionFilePath) => {
+const appendNotIncludePush = async (inputs, subscriptionFilePath, dlqTopic) => {
   const pushArray = [];
 
   pushArray.push(
@@ -56,11 +71,17 @@ const appendNotIncludePush = async (inputs, subscriptionFilePath) => {
     },
   );
 
+  await addDLQ(pushArray, inputs.env, dlqTopic);
   const yamlPushArray = yaml.dump(pushArray);
   fs.appendFileSync(subscriptionFilePath, `push_subscriptions:\n${yamlPushArray}`);
 };
 
-const appendIncludePush = async (fileContent, originalContentYaml, subscriptionFilePath, input) => {
+const appendIncludePush = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic) => {
   if (fileContent.includes('push_subscriptions')) {
     const pushArray = Object.values(originalContentYaml.push_subscriptions);
     const yamlPushArray = yaml.dump(pushArray);
@@ -78,6 +99,7 @@ const appendIncludePush = async (fileContent, originalContentYaml, subscriptionF
       expiration_policy: '',
     },
   );
+  await addDLQ(newPushArray, input.env, dlqTopic);
   const yamlPushArray = yaml.dump(newPushArray);
   fs.appendFileSync(subscriptionFilePath, yamlPushArray);
 

package/generators/resources/pubsub/handle-subscribers.js

@@ -4,7 +4,13 @@ const {
   appendNotIncludePull, appendIncludePull, appendNotIncludePush, appendIncludePush,
 } = require('./append');
 
-const handleSubscribers = async (env, answers, oidcEmail, pushEndpoint, subscriptionFilePath) => {
+const handleSubscribers = async (
+  env,
+  answers,
+  oidcEmail,
+  pushEndpoint,
+  subscriptionFilePath,
+  dlqTopic) => {
   const {
     subscriberName,
     existingTopic,
@@ -22,22 +28,35 @@ const handleSubscribers = async (env, answers, oidcEmail, pushEndpoint, subscrip
     audience,
     oidcEmail,
     pushEndpoint,
+    dlqTopic,
   };
 
   if (pushOrPull === 'pull') {
     if (subscriptionFileContent.length === 0 || !subscriptionFileContent.includes('pull_subscriptions')) {
-      await appendNotIncludePull(inputs, subscriptionFilePath);
+      await appendNotIncludePull(inputs, subscriptionFilePath, dlqTopic);
     } else {
       const originalContentYaml = yaml.load(subscriptionFileContent);
       const fileContent = subscriptionFileContent;
-      await appendIncludePull(fileContent, originalContentYaml, subscriptionFilePath, inputs);
+      await appendIncludePull(
+        fileContent,
+        originalContentYaml,
+        subscriptionFilePath,
+        inputs,
+        dlqTopic,
+      );
     }
   } else if (subscriptionFileContent.length === 0 || !subscriptionFileContent.includes('push_subscriptions')) {
-    await appendNotIncludePush(inputs, subscriptionFilePath);
+    await appendNotIncludePush(inputs, subscriptionFilePath, dlqTopic);
   } else {
     const originalContentYaml = yaml.load(subscriptionFileContent);
     const fileContent = subscriptionFileContent;
-    await appendIncludePush(fileContent, originalContentYaml, subscriptionFilePath, inputs);
+    await appendIncludePush(
+      fileContent,
+      originalContentYaml,
+      subscriptionFilePath,
+      inputs,
+      dlqTopic,
+    );
   }
 };
 

package/generators/resources/pubsub/index.js

@@ -170,6 +170,18 @@ module.exports = class extends BaseGenerator {
       externalSub,
     } = this.answers;
 
+    const dlqTopicName = `${getProjectId('prod').split('-')[0]}-common-dlq`;
+    let dlqTopic = `projects/${getProjectId('prod')}/topics/${dlqTopicName}`;
+
+    const dlqTopicDirPath = path.join(process.cwd(), 'infra', 'prod', 'pubsub', dlqTopicName);
+    this.fs.copyTpl(
+      this.templatePath('pubsub-dlq/terragrunt.hcl'),
+      this.destinationPath(`${dlqTopicDirPath}/terragrunt.hcl`),
+      {
+        ...this.answers,
+      },
+    );
+
     ['prod', 'staging'].forEach(async (env) => {
       if (createResource === 'topic') {
         const topicDirPath = path.join(process.cwd(), 'infra', env, 'pubsub', topicName);
@@ -182,6 +194,7 @@ module.exports = class extends BaseGenerator {
               env,
               topicName,
               subscriberName,
+              dlqTopic,
             },
           );
         });
@@ -212,6 +225,7 @@ module.exports = class extends BaseGenerator {
               audience,
               oidcEmail,
               pushEndpoint,
+              dlqTopic,
             },
           );
         }
@@ -228,7 +242,7 @@ module.exports = class extends BaseGenerator {
           );
         });
 
-        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, `${subscriptionDirPath}/subscribers.yaml`);
+        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, `${subscriptionDirPath}/subscribers.yaml`, dlqTopic);
       }
       if (createResource === 'subscription' && externalSub === 'yes') {
         const externalDirPath = path.join(process.cwd(), 'infra', env, 'pubsub', existingTopic, clanName);
@@ -259,6 +273,7 @@ module.exports = class extends BaseGenerator {
           oidcEmail = `${oidcName}@${stagingProjectIdConsumer}.iam.gserviceaccount.com`;
         } else {
           oidcEmail = `${oidcName}@${prodProjectIdConsumer}.iam.gserviceaccount.com`;
+          dlqTopic = `projects/${prodProjectIdConsumer}/topics/${prodProjectIdConsumer.split('-')[0]}-common-dlq`;
         }
         if (!fs.existsSync(externalSubPath)) {
           fs.writeFileSync(externalSubPath, '');
@@ -273,11 +288,19 @@ module.exports = class extends BaseGenerator {
               audience,
               oidcEmail,
               pushEndpoint,
+              dlqTopic,
             },
           );
         }
 
-        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, externalSubPath);
+        await handleSubscribers(
+          env,
+          this.answers,
+          oidcEmail,
+          pushEndpoint,
+          externalSubPath,
+          dlqTopic,
+        );
       }
     });
 

package/generators/resources/pubsub/templates/pubsub/subscribers.yaml

@@ -1,9 +1,11 @@
 <% if (createResource == 'topic') { %><% } %><% if (createResource == 'subscription' && pushOrPull == 'push') { %>push_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  push_endpoint: "<%-pushEndpoint%>"
+  push_endpoint: "<%-pushEndpoint%>" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   oidc_service_account_email: "<%-oidcEmail%>"
   audience: "<%-audience%>"
   expiration_policy: ""<% } %><% if (createResource == 'subscription' && pushOrPull == 'pull') { %>pull_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  ack_deadline_seconds: "60"
+  ack_deadline_seconds: "60" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   expiration_policy: ""<% } %>

package/generators/resources/pubsub/templates/pubsub-dlq/terragrunt.hcl

@@ -0,0 +1,41 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/terraform-google-modules/terraform-google-pubsub//?ref=v3.0.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+  common_vars  = read_terragrunt_config(find_in_parent_folders("common.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge (
+    local.project_vars.locals,
+    {
+      topic                = "${local.common_vars.locals.clan_name}-common-dlq"
+      create_subscriptions = true
+      create_topic         = true
+      push_subscriptions = [
+          {
+          name                        = "dlq-message-handler-subscription",
+          push_endpoint               = "https://europe-west1-sre-prod-5462.cloudfunctions.net/dlq-message-handler",
+          audience                    = "https://europe-west1-sre-prod-5462.cloudfunctions.net/dlq-message-handler",
+          expiration_policy           = "",
+          oidc_service_account_email  = "pubsub-dlq-handler@${local.project_vars.locals.project_id}.iam.gserviceaccount.com",
+          },
+        ],
+      topic_labels = {
+        cc = local.common_vars.locals.cost_center
+      }
+      subscription_labels = {
+        cc = local.common_vars.locals.cost_center
+      }
+      grant_token_creator = false,
+    }
+)

package/generators/resources/pubsub/templates/pubsub-external/subscribers.yaml

@@ -1,9 +1,11 @@
 <% if (pushOrPull == 'push') { %>push_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  push_endpoint: "<%-pushEndpoint%>"
+  push_endpoint: "<%-pushEndpoint%>" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   oidc_service_account_email: "<%-oidcEmail%>"
   audience: "<%-audience%>"
   expiration_policy: ""<% } %><% if (pushOrPull == 'pull') { %>pull_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  ack_deadline_seconds: "60"
+  ack_deadline_seconds: "60" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   expiration_policy: ""<% } %>

package/generators/resources/spanner/index.js

@@ -50,6 +50,11 @@ module.exports = class extends BaseGenerator {
         name: 'ddl',
         message: 'Please provide an optional list of DDL statements to run inside the newly created database. Leave empty to add them later.',
       },
+      {
+        type: 'input',
+        name: 'versionRetentionPeriod',
+        message: 'Please provide version retention period. Maximum value 7d, possible values include 84000s, 1h, 2d. Leave empty for default value 1h.',
+      },
     ];
 
     return this.prompt(prompts).then((props) => {
@@ -64,6 +69,7 @@ module.exports = class extends BaseGenerator {
       instanceAllocation,
       databaseName,
       ddl,
+      versionRetentionPeriod,
     } = this.answers;
 
     ['prod', 'staging'].forEach((env) => {
@@ -78,6 +84,7 @@ module.exports = class extends BaseGenerator {
           instanceAllocation,
           databaseName,
           ddl,
+          versionRetentionPeriod,
         },
       );
     });

package/generators/resources/spanner/templates/spanner/spec.hcl

@@ -17,4 +17,8 @@ locals {
   # The number of processing units allocated to this instance.
   # At most one of either num_nodes or processing_units can be present in terraform.
   processing_units = "<%-processingUnits%>" <% } %>
+  # The time frame it will be possible to restore the database to a point in time.
+  # Values from 1h up to max 7d
+  <% if (versionRetentionPeriod !== '') { %>
+  version_retention_period = "<%-versionRetentionPeriod%>" <% } %>
 }

package/generators/resources/spanner/templates/spanner/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/extenda/tf-module-gcp-spanner//?ref=v0.1.0"
+  source = "git::https://github.com/extenda/tf-module-gcp-spanner//?ref=v0.1.1"
 }
 
 # Include all settings from the root terragrunt.hcl file

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.74.0",
+  "version": "0.75.2",
   "description": "Infrastructure as code generator for GCP.",
   "main": "src/cli.js",
   "bin": {