@hiiretail/gcp-infra-cli 0.72.0 → 0.75.0

package/generators/resources/monitoring/append.js

@@ -10,57 +10,68 @@ const appendIncludeConfigSlo = async (fileContent, originalContentYaml, slosFile
 
   const newPullArray = [];
 
+  const availabilityConf = {
+    display_name: 'Month - Availability',
+    slo_id: 'month-availability',
+    goal: 0.998,
+    calendar_period: 'MONTH',
+    type: 'windows_based_sli',
+    method: 'boolean_filter',
+    window_period: '60s',
+  };
+
   if (inputs.sli === 'availability') {
-    newPullArray.push(
-      {
-        display_name: 'Month - Availability',
-        slo_id: 'month-availability',
-        goal: 0.998,
-        calendar_period: 'MONTH',
-        type: 'windows_based_sli',
-        method: 'boolean_filter',
-        window_period: '60s',
-      },
-    );
+    if (inputs.burnRateAlerts === 'no') {
+      availabilityConf.alert = {};
+    }
+    newPullArray.push(availabilityConf);
   }
-  if (inputs.sli === 'error-rate') {
-    newPullArray.push(
-      {
-        display_name: 'Month - Error rate',
-        slo_id: 'month-error-rate',
-        goal: 0.999,
-        calendar_period: 'MONTH',
-        type: 'request_based_sli',
-        method: 'good_total_ratio',
-        bad_service_filter:
+
+  const errorRateConf = {
+    display_name: 'Month - Error rate',
+    slo_id: 'month-error-rate',
+    goal: 0.999,
+    calendar_period: 'MONTH',
+    type: 'request_based_sli',
+    method: 'good_total_ratio',
+    bad_service_filter:
 `metric.type="knative.dev/serving/revision/request_count"
 resource.type="knative_revision"
 metric.labels.response_code_class="5xx"
 resource.labels.service_name="${inputs.serviceName}"`,
-        total_service_filter:
+    total_service_filter:
 `metric.type="knative.dev/serving/revision/request_count"
 resource.type="knative_revision"
 resource.labels.service_name=${inputs.serviceName}"`,
-      },
-    );
+  };
+
+  if (inputs.sli === 'error-rate') {
+    if (inputs.burnRateAlerts === 'no') {
+      errorRateConf.alert = {};
+    }
+    newPullArray.push(errorRateConf);
   }
-  if (inputs.sli === 'latency') {
-    newPullArray.push(
-      {
-        display_name: 'Month - Latency',
-        slo_id: 'month-latency',
-        goal: 0.95,
-        calendar_period: 'MONTH',
-        type: 'request_based_sli',
-        method: 'distribution_cut',
-        metric_filter:
+
+  const latencyConf = {
+    display_name: 'Month - Latency',
+    slo_id: 'month-latency',
+    goal: 0.95,
+    calendar_period: 'MONTH',
+    type: 'request_based_sli',
+    method: 'distribution_cut',
+    metric_filter:
 `metric.type="knative.dev/serving/revision/request_latencies"
 resource.type="knative_revision"
 resource.labels.service_name="${inputs.serviceName}"`,
-        range_min: 0,
-        range_max: 100,
-      },
-    );
+    range_min: 0,
+    range_max: 100,
+  };
+
+  if (inputs.sli === 'latency') {
+    if (inputs.burnRateAlerts === 'no') {
+      latencyConf.alert = {};
+    }
+    newPullArray.push(latencyConf);
   }
 
   const finalYamlPullArray = yaml.dump(newPullArray);

package/generators/resources/monitoring/handle-alerts.js

@@ -0,0 +1,11 @@
+const ejs = require('ejs');
+
+const handleAlerts = (alerts, templates, answers) => {
+  const template = templates[`${answers.alertResource}`][`${answers.alert}`];
+  const newAlert = ejs.render(JSON.stringify(template), answers);
+
+  alerts.push(JSON.parse(newAlert));
+  return alerts;
+};
+
+module.exports = handleAlerts;

package/generators/resources/monitoring/handle-slos.js

@@ -7,6 +7,7 @@ const handleSlosFile = async (answers, slosFilePath) => {
     serviceName,
     sli,
     systemName,
+    burnRateAlerts,
   } = answers;
 
   const sloFileContent = fs.readFileSync(slosFilePath, 'utf8');
@@ -16,6 +17,7 @@ const handleSlosFile = async (answers, slosFilePath) => {
     serviceName,
     sli,
     systemName,
+    burnRateAlerts,
   };
 
   const originalContentYaml = yaml.load(sloFileContent);

package/generators/resources/monitoring/index.js

@@ -1,36 +1,86 @@
 const path = require('path');
 const chalk = require('chalk');
 const fs = require('fs');
+const yaml = require('js-yaml');
 const BaseGenerator = require('../../../src/BaseGenerator');
 const { required } = require('../../../src/validators');
 const helper = require('./validate');
 const handleSlosFile = require('./handle-slos');
 const handleUptimeFile = require('./handle-uptime');
+const handleAlerts = require('./handle-alerts');
+
+const alertTemplates = yaml.load(fs.readFileSync(`${__dirname}/templates/alerts/alerts.yaml`));
 
 module.exports = class extends BaseGenerator {
-  prompting() {
-    const prompts = [
+  async prompting() {
+    this.answers = await this.prompt([
       {
         type: 'list',
         name: 'monitoringResource',
         message: 'Select the resource you want to create',
-        default: 'uptime-checks',
-        choices: ['uptime-checks', 'slos'],
+        choices: ['alerts', 'uptime-checks', 'slos'],
+      },
+      {
+        when: (response) => response.monitoringResource === 'alerts',
+        type: 'list',
+        name: 'alertResource',
+        choices: Object.keys(alertTemplates),
+      },
+      {
+        when: (response) => response.monitoringResource === 'alerts',
+        type: 'list',
+        name: 'alert',
+        choices: (answers) => Object.keys(alertTemplates[`${answers.alertResource}`]),
       },
       {
-        when: (response) => response.monitoringResource === 'uptime-checks' || 'slos',
+        when: (response) => response.monitoringResource === ('alerts' || 'slos' || 'uptime-checks'),
         type: 'input',
         name: 'systemName',
         message: 'Please provide three-letter system name as defined in Styra',
         validate: required && helper.validSystemName,
       },
       {
-        when: (response) => response.monitoringResource === 'uptime-checks' || 'slos',
+        when: (response) => response.monitoringResource === ('slos' || 'uptime-checks') || response.alertResource === 'cloud_run',
         type: 'input',
         name: 'serviceName',
         message: 'Please provide the namespace where the service resides',
         validate: required,
       },
+      {
+        when: (response) => response.monitoringResource === 'alerts',
+        type: 'input',
+        name: 'runbookLink',
+        message: 'Please provide the full URL to your runbook in confluence (Leave empty if none)',
+        validate: required && helper.validUrl,
+      },
+      {
+        when: (response) => response.alertResource === 'cloud_scheduler',
+        type: 'input',
+        name: 'jobId',
+        message: 'Please provide the "job id"',
+        validate: required,
+      },
+      {
+        when: (response) => response.alertResource === 'cloud_sql',
+        type: 'input',
+        name: 'databaseId',
+        message: 'Please provide the "database id"',
+        validate: required,
+      },
+      {
+        when: (response) => response.alertResource === 'memorystore',
+        type: 'input',
+        name: 'instanceId',
+        message: 'Please provide the "instance id"',
+        validate: required,
+      },
+      {
+        when: (response) => response.alertResource === 'pub_sub',
+        type: 'input',
+        name: 'subscriptionId',
+        message: 'Please provide the "subscription id"',
+        validate: required,
+      },
       {
         when: (response) => response.monitoringResource === 'uptime-checks',
         type: 'input',
@@ -50,20 +100,23 @@ module.exports = class extends BaseGenerator {
         type: 'list',
         name: 'sli',
         message: 'Please select the SLI',
-        default: 'availability',
         choices: ['availability', 'error-rate', 'latency'],
       },
+      {
+        when: (response) => response.monitoringResource === 'slos',
+        type: 'list',
+        name: 'burnRateAlerts',
+        message: 'Please select yes if you want to create burn rate alert for the SLI',
+        default: 'yes',
+        choices: ['yes', 'no'],
+      },
       {
         when: (response) => response.monitoringResource === 'slos' && response.sli === 'availability',
         type: 'confirm',
         name: 'info',
         message: 'WARNING: Make sure that an uptime check has been created before applying availability SLI',
       },
-    ];
-
-    return this.prompt(prompts).then((props) => {
-      this.answers = props;
-    });
+    ]);
   }
 
   async writing() {
@@ -73,22 +126,21 @@ module.exports = class extends BaseGenerator {
       hostname,
       sli,
       systemName,
+      burnRateAlerts,
     } = this.answers;
 
-    const serviceFolderName = serviceName.replace(/ /g, '-').toLowerCase();
-    const serviceDir = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource, serviceFolderName);
-    const uptimeDirPath = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource);
+    const resourcePath = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource);
 
     if (monitoringResource === 'uptime-checks') {
-      if (!fs.existsSync(uptimeDirPath)) {
-        fs.mkdirSync(uptimeDirPath, { recursive: true });
+      if (!fs.existsSync(resourcePath)) {
+        fs.mkdirSync(resourcePath, { recursive: true });
       }
 
-      const uptimeYamlFile = `${uptimeDirPath}/uptime-checks.yaml`;
+      const uptimeYamlFile = `${resourcePath}/uptime-checks.yaml`;
       if (!fs.existsSync(uptimeYamlFile)) {
         this.copyDir(
           'uptime-checks',
-          uptimeDirPath,
+          resourcePath,
           {
             ...this.answers,
             serviceName,
@@ -102,6 +154,8 @@ module.exports = class extends BaseGenerator {
     }
 
     if (monitoringResource === 'slos') {
+      const serviceFolderName = serviceName.replace(/ /g, '-').toLowerCase();
+      const serviceDir = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource, serviceFolderName);
       const fileContainsFilter = (fileName, str) => {
         const contents = fs.readFileSync(fileName, 'utf-8');
         const result = contents.includes(str);
@@ -122,6 +176,7 @@ module.exports = class extends BaseGenerator {
               monitoringResource,
               serviceName,
               systemName,
+              burnRateAlerts,
             },
           );
         }
@@ -134,6 +189,7 @@ module.exports = class extends BaseGenerator {
             monitoringResource,
             serviceName,
             systemName,
+            burnRateAlerts,
           },
         );
       }
@@ -149,12 +205,33 @@ module.exports = class extends BaseGenerator {
             serviceName,
             systemName,
             sli,
+            burnRateAlerts,
           },
         );
       } else {
         await handleSlosFile(this.answers, sloYamlFile);
       }
     }
+
+    if (monitoringResource === 'alerts') {
+      const yamlPath = `${resourcePath}/alerts.yaml`;
+      const terraPath = `${resourcePath}/terragrunt.hcl`;
+      if (!fs.existsSync(resourcePath)) fs.mkdirSync(resourcePath, { recursive: true });
+      if (!fs.existsSync(yamlPath)) fs.writeFileSync(yamlPath, '');
+
+      if (!fs.existsSync(terraPath)) {
+        this.fs.copyTpl(
+          this.templatePath('alerts/terragrunt.hcl'),
+          this.destinationPath(terraPath),
+          this.answers,
+        );
+      }
+
+      const oldYaml = yaml.load(fs.readFileSync(yamlPath, 'utf8')) || [];
+      const newYaml = await handleAlerts(oldYaml, alertTemplates, this.answers);
+
+      fs.writeFileSync(yamlPath, yaml.dump(newYaml));
+    }
   }
 
   end() {

package/generators/resources/monitoring/templates/alerts/alerts.yaml

@@ -0,0 +1,257 @@
+cloud_run:
+  error_count:
+    display_name: "[P3] <%-systemName%>.<%-serviceName%> | 5xx Error Request Count above 1"
+    conditions:
+      - display_name: Cloud Run Anthos - 5xx error Request Count above 1
+        condition_threshold:
+          filter: |
+            resource.type="knative_revision"
+            resource.labels.service_name="<%-serviceName%>"
+            metric.type="knative.dev/serving/revision/request_count"
+            metric.labels.response_code_class="5xx"
+          threshold_value: 1
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_SUM
+              group_by_fields:
+                - metric.label.response_code_class
+              per_series_aligner: ALIGN_DELTA
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  error_rate:
+    display_name: "[P3] <%-systemName%>.<%-serviceName%> | High 5xx Error Rate"
+    conditions:
+      - display_name: Cloud Run Anthos - 3% of all requests during 10min are 5xx
+        condition_monitoring_query_language:
+          query: |
+            fetch knative_revision::knative.dev/serving/revision/request_count
+              | filter service_name = "store-data-resolver"
+              | align int_mean_aligner(10m)
+              | group_by [],    sum(if(metric.response_code_class == '5xx', val(), 0)) / sum(val())
+              | condition val() > 0.03
+              | every 10m
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  request_latency:
+    display_name: "[P3] <%-systemName%>.<%-serviceName%> | High Request Latency"
+    conditions:
+      - display_name: Cloud Run Anthos - Response Time (95%) above 1s for 5 min
+        condition_threshold:
+          filter: |
+            resource.type="knative_revision"
+            resource.labels.service_name="<%-serviceName%>"
+            metric.type="knative.dev/serving/revision/request_latencies"
+          threshold_value: 1000
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_PERCENTILE_95
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+cloud_scheduler:
+  failed_job:
+    display_name: "[P4] <%-systemName%> - Cloud Scheduler | <%-jobId%> - Job Failed"
+    conditions:
+      - display_name: Cloud Scheduler Job - Log entries with SEVERITY=Error exceed threshold
+        condition_threshold:
+          filter: |
+            resource.type="cloud_scheduler_job"
+            resource.labels.job_id="<%-jobId%>"
+            metric.type="logging.googleapis.com/log_entry_count"
+            metric.labels.severity="ERROR"
+          threshold_value: 1
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_COUNT
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+cloud_sql:
+  cpu_over_65:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - CPU over 65%"
+    conditions:
+      - display_name: Cloud SQL Database - CPU utilization above 65% over 5 min
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_database"
+            resource.labels.database_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/cpu/utilization"
+          threshold_value: 0.65
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  cpu_over_85:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - CPU over 85%"
+    conditions:
+      - display_name: "Cloud SQL Database - CPU-usage above 85% over 1 min"
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_database"
+            resource.labels.database_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/cpu/utilization"
+          threshold_value: 0.85
+          duration: 60s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  cpu_over_90:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - CPU over 90%"
+    conditions:
+      - display_name: Cloud SQL Database - CPU-usage above 90%
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_database"
+            resource.labels.database_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/cpu/utilization"
+          threshold_value: 0.9
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_50:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - Memory over 50%"
+    conditions:
+    - display_name: Cloud SQL Database - Memory utilization above 50% over 5 min
+      condition_threshold:
+        filter: |
+          resource.type="cloudsql_database"
+          resource.labels.database_id="<%-databaseId%>"
+          metric.type="cloudsql.googleapis.com/database/memory/utilization"
+        threshold_value: 50
+        duration: 300s
+        aggregations:
+        - alignment_period: 60s
+          cross_series_reducer: REDUCE_NONE
+          per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_75:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - Memory over 75%"
+    conditions:
+    - display_name: Cloud SQL Database - Memory utilization above 75% over 5 min
+      condition_threshold:
+        filter: |
+          resource.type="cloudsql_database"
+          resource.labels.database_id="<%-databaseId%>"
+          metric.type="cloudsql.googleapis.com/database/memory/utilization"
+        threshold_value: 75
+        duration: 300s
+        aggregations:
+        - alignment_period: 60s
+          cross_series_reducer: REDUCE_NONE
+          per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_90:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - Memory over 90%"
+    conditions:
+    - display_name: Cloud SQL Database - Memory utilization above 90%
+      condition_threshold:
+        filter: |
+          resource.type="cloudsql_database"
+          resource.labels.database_id="<%-databaseId%>"
+          metric.type="cloudsql.googleapis.com/database/memory/utilization"
+        threshold_value: 90
+        duration: 60s
+        aggregations:
+        - alignment_period: 60s
+          cross_series_reducer: REDUCE_NONE
+          per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  query_over_1s:
+    display_name: "[P4] <%-systemName%> - CloudSQL | <%-databaseId%> - Query resolve time"
+    conditions:
+      - display_name: Cloud SQL Instance Database - Per query execution times above 1000 ms
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_instance_database"
+            resource.labels.resource_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/postgresql/insights/perquery/execution_time"
+          threshold_value: 1000000
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_DELTA
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+memorystore:
+  memory_over_50:
+    display_name: "[P4] <%-systemName%> - Memorystore | <%-instanceId%> - Memory over 50%"
+    conditions:
+      - display_name: Memorystore Redis Instance - Memory Usage above 50% over 5 min
+        condition_threshold:
+          filter: |
+            resource.type="redis_instance"
+            resource.labels.instance_id="<%-instanceId%>"
+            metric.type="redis.googleapis.com/stats/memory/usage_ratio"
+          threshold_value: 0.5
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_75:
+    display_name: "[P4] <%-systemName%> - Memorystore | <%-instanceId%> - Memory over 75%"
+    conditions:
+      - display_name: Memorystore Redis Instance - Memory Usage above 75% for 5min
+        condition_threshold:
+          filter: |
+            resource.type="redis_instance"
+            resource.labels.instance_id="<%-instanceId%>"
+            metric.type="redis.googleapis.com/stats/memory/usage_ratio"
+          threshold_value: 0.75
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_90:
+    display_name: "[P2] <%-systemName%> - Memorystore | <%-instanceId%> - Memory over 90%"
+    conditions:
+      - display_name: Memorystore Redis Instance - Memory Usage above 90%
+        condition_threshold:
+          filter: |
+            resource.type="redis_instance"
+            resource.labels.instance_id="<%-instanceId%>"
+            metric.type="redis.googleapis.com/stats/memory/usage_ratio"
+          threshold_value: 0.90
+          duration: 60s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+pub_sub:
+  unacknowledged_messages:
+    display_name: "[P4] <%-systemName%> - Pub/Sub | <%-subscriptionId%> - Undelivered message(s)"
+    conditions:
+      - display_name: Cloud Pub/Sub Subscription - Undelivered messages above 1 for 5 min
+        condition_threshold:
+          filter: |
+            resource.type="pubsub_subscription"
+            resource.labels.subscription_id="<%-subscriptionId%>"
+            metric.type="pubsub.googleapis.com/subscription/num_undelivered_messages"
+          threshold_value: 1
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MEAN
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>

package/generators/resources/monitoring/templates/alerts/terragrunt.hcl

@@ -0,0 +1,33 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/extenda/tf-module-gcp-alert-policy//?ref=v0.1.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+dependency "notification_channels" {
+  config_path  = "../notification-channels"
+  mock_outputs = {
+    notification_channels = ["dummy-channel"]
+  }
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+  common_vars  = read_terragrunt_config(find_in_parent_folders("common.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = {
+  monitoring_project_id   = local.project_vars.locals.monitoring_project_id,
+  notification_channels   = dependency.notification_channels.outputs.notification_channels,
+  policies                = yamldecode(file("${get_terragrunt_dir()}/alerts.yaml")),
+  user_labels             = {
+    cc   = local.common_vars.locals.cost_center
+    clan = local.common_vars.locals.clan_name
+  },
+}

package/generators/resources/monitoring/templates/slos/slos.yaml

@@ -9,13 +9,15 @@
     resource.type="knative_revision"
     resource.labels.service_name="<%-serviceName%>"
   range_min: 0
-  range_max: 100<% } %><% if (sli === 'availability') { %>- display_name: Month - Availability
+  range_max: 100<% if (burnRateAlerts === 'no') { %>
+  alert: {}<% } %><% } %><% if (sli === 'availability') { %>- display_name: Month - Availability
   slo_id: month-availability
   goal: 0.998
   calendar_period: MONTH
   type: windows_based_sli
   method: boolean_filter
-  window_period: 60s<% } %><% if (sli === 'error-rate') { %>- display_name: Month - Error rate
+  window_period: 60s<% if (burnRateAlerts === 'no') { %>
+  alert: {}<% } %><% } %><% if (sli === 'error-rate') { %>- display_name: Month - Error rate
   slo_id: month-error-rate
   goal: 0.999
   calendar_period: MONTH
@@ -29,4 +31,5 @@
   total_service_filter: |-
     metric.type="knative.dev/serving/revision/request_count"
     resource.type="knative_revision"
-    resource.labels.service_name="<%-serviceName%>"<% } %>
+    resource.labels.service_name="<%-serviceName%>"<% if (burnRateAlerts === 'no') { %>
+  alert: {}<% } %><% } %>

package/generators/resources/monitoring/templates/slos/terragrunt.hcl

@@ -24,8 +24,9 @@ locals {
 inputs = merge(
   local.project_vars.locals,
   {
-    service_name          = "<%-systemName%>.<%-serviceName%>"
-    slos                  = yamldecode(file("${get_terragrunt_dir()}/slos.yaml")),
+    service_name            = "<%-systemName%>.<%-serviceName%>"
+    slos                    = yamldecode(file("${get_terragrunt_dir()}/slos.yaml")),
+    telemetry_resource_name = "//container.googleapis.com/projects/${local.project_vars.locals.monitoring_project_id}/locations/europe-west1/clusters/k8s-cluster/k8s/namespaces/<%-serviceName%>"
     <% if (sli === 'availability') { %>
     metric_filter = {
       "metric.type"            = "monitoring.googleapis.com/uptime_check/check_passed"

package/generators/resources/monitoring/templates/uptime-checks/uptime-checks.yaml

@@ -1,3 +1,3 @@
-- service_name: "<%-systemName%>.<%-serviceName%>"
-  hostname: "<%-hostname%>"
-  path: "<%-path%>"
+- service_name: <%-systemName%>.<%-serviceName%>
+  hostname: <%-hostname%>
+  path: <%-path%>

package/generators/resources/monitoring/validate.js

@@ -15,4 +15,11 @@ helper.validSystemName = (input) => {
   return 'System name must be 3 characters';
 };
 
+helper.validUrl = (input) => {
+  // eslint-disable-next-line no-useless-escape
+  const regex = new RegExp(/^https:\/\/[a-zA-Z]*.[a-zA-Z]*.[a-zA-Z]*\/[a-zA-Z\/+_-]*.$/g);
+  if (regex.test(input) || input === '') return true;
+  return 'Enter a valid URL';
+};
+
 module.exports = helper;

package/generators/resources/pubsub/append.js

@@ -1,7 +1,15 @@
 const yaml = require('js-yaml');
 const fs = require('fs');
 
-const appendNotIncludePull = async (inputs, subscriptionFilePath) => {
+const addDLQ = async (yamlArray, env, dlqTopic) => {
+  if (env === 'prod') {
+    /* eslint-disable */
+    yamlArray[0].dead_letter_topic = dlqTopic;
+    /* eslint-enable */
+  }
+};
+
+const appendNotIncludePull = async (inputs, subscriptionFilePath, dlqTopic) => {
   const pullArray = [];
 
   pullArray.push(
@@ -12,11 +20,17 @@ const appendNotIncludePull = async (inputs, subscriptionFilePath) => {
     },
   );
 
+  await addDLQ(pullArray, inputs.env, dlqTopic);
   const yamlPullArray = yaml.dump(pullArray);
   fs.appendFileSync(subscriptionFilePath, `pull_subscriptions:\n${yamlPullArray}`);
 };
 
-const appendIncludePull = async (fileContent, originalContentYaml, subscriptionFilePath, input) => {
+const appendIncludePull = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic) => {
   if (fileContent.includes('pull_subscriptions')) {
     const pullArray = Object.values(originalContentYaml.pull_subscriptions);
     const yamlPullArray = yaml.dump(pullArray);
@@ -33,6 +47,7 @@ const appendIncludePull = async (fileContent, originalContentYaml, subscriptionF
     },
   );
 
+  await addDLQ(newPullArray, input.env, dlqTopic);
   const finalYamlPullArray = yaml.dump(newPullArray);
   fs.appendFileSync(subscriptionFilePath, finalYamlPullArray);
 
@@ -43,7 +58,7 @@ const appendIncludePull = async (fileContent, originalContentYaml, subscriptionF
   }
 };
 
-const appendNotIncludePush = async (inputs, subscriptionFilePath) => {
+const appendNotIncludePush = async (inputs, subscriptionFilePath, dlqTopic) => {
   const pushArray = [];
 
   pushArray.push(
@@ -56,11 +71,17 @@ const appendNotIncludePush = async (inputs, subscriptionFilePath) => {
     },
   );
 
+  await addDLQ(pushArray, inputs.env, dlqTopic);
   const yamlPushArray = yaml.dump(pushArray);
   fs.appendFileSync(subscriptionFilePath, `push_subscriptions:\n${yamlPushArray}`);
 };
 
-const appendIncludePush = async (fileContent, originalContentYaml, subscriptionFilePath, input) => {
+const appendIncludePush = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic) => {
   if (fileContent.includes('push_subscriptions')) {
     const pushArray = Object.values(originalContentYaml.push_subscriptions);
     const yamlPushArray = yaml.dump(pushArray);
@@ -78,6 +99,7 @@ const appendIncludePush = async (fileContent, originalContentYaml, subscriptionF
       expiration_policy: '',
     },
   );
+  await addDLQ(newPushArray, input.env, dlqTopic);
   const yamlPushArray = yaml.dump(newPushArray);
   fs.appendFileSync(subscriptionFilePath, yamlPushArray);
 

package/generators/resources/pubsub/handle-subscribers.js

@@ -4,7 +4,13 @@ const {
   appendNotIncludePull, appendIncludePull, appendNotIncludePush, appendIncludePush,
 } = require('./append');
 
-const handleSubscribers = async (env, answers, oidcEmail, pushEndpoint, subscriptionFilePath) => {
+const handleSubscribers = async (
+  env,
+  answers,
+  oidcEmail,
+  pushEndpoint,
+  subscriptionFilePath,
+  dlqTopic) => {
   const {
     subscriberName,
     existingTopic,
@@ -22,22 +28,35 @@ const handleSubscribers = async (env, answers, oidcEmail, pushEndpoint, subscrip
     audience,
     oidcEmail,
     pushEndpoint,
+    dlqTopic,
   };
 
   if (pushOrPull === 'pull') {
     if (subscriptionFileContent.length === 0 || !subscriptionFileContent.includes('pull_subscriptions')) {
-      await appendNotIncludePull(inputs, subscriptionFilePath);
+      await appendNotIncludePull(inputs, subscriptionFilePath, dlqTopic);
     } else {
       const originalContentYaml = yaml.load(subscriptionFileContent);
       const fileContent = subscriptionFileContent;
-      await appendIncludePull(fileContent, originalContentYaml, subscriptionFilePath, inputs);
+      await appendIncludePull(
+        fileContent,
+        originalContentYaml,
+        subscriptionFilePath,
+        inputs,
+        dlqTopic,
+      );
     }
   } else if (subscriptionFileContent.length === 0 || !subscriptionFileContent.includes('push_subscriptions')) {
-    await appendNotIncludePush(inputs, subscriptionFilePath);
+    await appendNotIncludePush(inputs, subscriptionFilePath, dlqTopic);
   } else {
     const originalContentYaml = yaml.load(subscriptionFileContent);
     const fileContent = subscriptionFileContent;
-    await appendIncludePush(fileContent, originalContentYaml, subscriptionFilePath, inputs);
+    await appendIncludePush(
+      fileContent,
+      originalContentYaml,
+      subscriptionFilePath,
+      inputs,
+      dlqTopic,
+    );
   }
 };
 

package/generators/resources/pubsub/index.js

@@ -170,6 +170,18 @@ module.exports = class extends BaseGenerator {
       externalSub,
     } = this.answers;
 
+    const dlqTopicName = `${getProjectId('prod').split('-')[0]}-common-dlq`;
+    let dlqTopic = `projects/${getProjectId('prod')}/topics/${dlqTopicName}`;
+
+    const dlqTopicDirPath = path.join(process.cwd(), 'infra', 'prod', 'pubsub', dlqTopicName);
+    this.fs.copyTpl(
+      this.templatePath('pubsub-dlq/terragrunt.hcl'),
+      this.destinationPath(`${dlqTopicDirPath}/terragrunt.hcl`),
+      {
+        ...this.answers,
+      },
+    );
+
     ['prod', 'staging'].forEach(async (env) => {
       if (createResource === 'topic') {
         const topicDirPath = path.join(process.cwd(), 'infra', env, 'pubsub', topicName);
@@ -182,6 +194,7 @@ module.exports = class extends BaseGenerator {
               env,
               topicName,
               subscriberName,
+              dlqTopic,
             },
           );
         });
@@ -212,6 +225,7 @@ module.exports = class extends BaseGenerator {
               audience,
               oidcEmail,
               pushEndpoint,
+              dlqTopic,
             },
           );
         }
@@ -228,7 +242,7 @@ module.exports = class extends BaseGenerator {
           );
         });
 
-        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, `${subscriptionDirPath}/subscribers.yaml`);
+        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, `${subscriptionDirPath}/subscribers.yaml`, dlqTopic);
       }
       if (createResource === 'subscription' && externalSub === 'yes') {
         const externalDirPath = path.join(process.cwd(), 'infra', env, 'pubsub', existingTopic, clanName);
@@ -259,6 +273,7 @@ module.exports = class extends BaseGenerator {
           oidcEmail = `${oidcName}@${stagingProjectIdConsumer}.iam.gserviceaccount.com`;
         } else {
           oidcEmail = `${oidcName}@${prodProjectIdConsumer}.iam.gserviceaccount.com`;
+          dlqTopic = `projects/${prodProjectIdConsumer}/topics/${prodProjectIdConsumer.split('-')[0]}-common-dlq`;
         }
         if (!fs.existsSync(externalSubPath)) {
           fs.writeFileSync(externalSubPath, '');
@@ -273,11 +288,19 @@ module.exports = class extends BaseGenerator {
               audience,
               oidcEmail,
               pushEndpoint,
+              dlqTopic,
             },
           );
         }
 
-        await handleSubscribers(env, this.answers, oidcEmail, pushEndpoint, externalSubPath);
+        await handleSubscribers(
+          env,
+          this.answers,
+          oidcEmail,
+          pushEndpoint,
+          externalSubPath,
+          dlqTopic,
+        );
       }
     });
 

package/generators/resources/pubsub/templates/pubsub/subscribers.yaml

@@ -1,9 +1,11 @@
 <% if (createResource == 'topic') { %><% } %><% if (createResource == 'subscription' && pushOrPull == 'push') { %>push_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  push_endpoint: "<%-pushEndpoint%>"
+  push_endpoint: "<%-pushEndpoint%>" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   oidc_service_account_email: "<%-oidcEmail%>"
   audience: "<%-audience%>"
   expiration_policy: ""<% } %><% if (createResource == 'subscription' && pushOrPull == 'pull') { %>pull_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  ack_deadline_seconds: "60"
+  ack_deadline_seconds: "60" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   expiration_policy: ""<% } %>

package/generators/resources/pubsub/templates/pubsub-dlq/terragrunt.hcl

@@ -0,0 +1,41 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/terraform-google-modules/terraform-google-pubsub//?ref=v3.0.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+  common_vars  = read_terragrunt_config(find_in_parent_folders("common.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge (
+    local.project_vars.locals,
+    {
+      topic                = "${local.common_vars.locals.clan_name}-common-dlq"
+      create_subscriptions = true
+      create_topic         = true
+      push_subscriptions = [
+          {
+          name                        = "dlq-message-handler-subscription",
+          push_endpoint               = "https://europe-west1-sre-prod-5462.cloudfunctions.net/dlq-message-handler",
+          audience                    = "https://europe-west1-sre-prod-5462.cloudfunctions.net/dlq-message-handler",
+          expiration_policy           = "",
+          oidc_service_account_email  = "pubsub-dlq-handler@${local.project_vars.locals.project_id}.iam.gserviceaccount.com",
+          },
+        ],
+      topic_labels = {
+        cc = local.common_vars.locals.cost_center
+      }
+      subscription_labels = {
+        cc = local.common_vars.locals.cost_center
+      }
+      grant_token_creator = false,
+    }
+)

package/generators/resources/pubsub/templates/pubsub-external/subscribers.yaml

@@ -1,9 +1,11 @@
 <% if (pushOrPull == 'push') { %>push_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  push_endpoint: "<%-pushEndpoint%>"
+  push_endpoint: "<%-pushEndpoint%>" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   oidc_service_account_email: "<%-oidcEmail%>"
   audience: "<%-audience%>"
   expiration_policy: ""<% } %><% if (pushOrPull == 'pull') { %>pull_subscriptions:
 - name: "<%-existingTopic%>+<%-subscriberName%>"
-  ack_deadline_seconds: "60"
+  ack_deadline_seconds: "60" <% if (env == 'prod') { %>
+  dead_letter_topic: "<%-dlqTopic%>" <% } %>
   expiration_policy: ""<% } %>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.72.0",
+  "version": "0.75.0",
   "description": "Infrastructure as code generator for GCP.",
   "main": "src/cli.js",
   "bin": {