@hiiretail/gcp-infra-cli 0.73.0 → 0.74.0

package/generators/resources/monitoring/handle-alerts.js

@@ -0,0 +1,11 @@
+const ejs = require('ejs');
+
+const handleAlerts = (alerts, templates, answers) => {
+  const template = templates[`${answers.alertResource}`][`${answers.alert}`];
+  const newAlert = ejs.render(JSON.stringify(template), answers);
+
+  alerts.push(JSON.parse(newAlert));
+  return alerts;
+};
+
+module.exports = handleAlerts;

package/generators/resources/monitoring/index.js

@@ -1,36 +1,86 @@
 const path = require('path');
 const chalk = require('chalk');
 const fs = require('fs');
+const yaml = require('js-yaml');
 const BaseGenerator = require('../../../src/BaseGenerator');
 const { required } = require('../../../src/validators');
 const helper = require('./validate');
 const handleSlosFile = require('./handle-slos');
 const handleUptimeFile = require('./handle-uptime');
+const handleAlerts = require('./handle-alerts');
+
+const alertTemplates = yaml.load(fs.readFileSync(`${__dirname}/templates/alerts/alerts.yaml`));
 
 module.exports = class extends BaseGenerator {
-  prompting() {
-    const prompts = [
+  async prompting() {
+    this.answers = await this.prompt([
       {
         type: 'list',
         name: 'monitoringResource',
         message: 'Select the resource you want to create',
-        default: 'uptime-checks',
-        choices: ['uptime-checks', 'slos'],
+        choices: ['alerts', 'uptime-checks', 'slos'],
       },
       {
-        when: (response) => response.monitoringResource === 'uptime-checks' || 'slos',
+        when: (response) => response.monitoringResource === 'alerts',
+        type: 'list',
+        name: 'alertResource',
+        choices: Object.keys(alertTemplates),
+      },
+      {
+        when: (response) => response.monitoringResource === 'alerts',
+        type: 'list',
+        name: 'alert',
+        choices: (answers) => Object.keys(alertTemplates[`${answers.alertResource}`]),
+      },
+      {
+        when: (response) => response.monitoringResource === ('alerts' || 'slos' || 'uptime-checks'),
         type: 'input',
         name: 'systemName',
         message: 'Please provide three-letter system name as defined in Styra',
         validate: required && helper.validSystemName,
       },
       {
-        when: (response) => response.monitoringResource === 'uptime-checks' || 'slos',
+        when: (response) => response.monitoringResource === ('slos' || 'uptime-checks') || response.alertResource === 'cloud_run',
         type: 'input',
         name: 'serviceName',
         message: 'Please provide the namespace where the service resides',
         validate: required,
       },
+      {
+        when: (response) => response.monitoringResource === 'alerts',
+        type: 'input',
+        name: 'runbookLink',
+        message: 'Please provide the full URL to your runbook in confluence (Leave empty if none)',
+        validate: required && helper.validUrl,
+      },
+      {
+        when: (response) => response.alertResource === 'cloud_scheduler',
+        type: 'input',
+        name: 'jobId',
+        message: 'Please provide the "job id"',
+        validate: required,
+      },
+      {
+        when: (response) => response.alertResource === 'cloud_sql',
+        type: 'input',
+        name: 'databaseId',
+        message: 'Please provide the "database id"',
+        validate: required,
+      },
+      {
+        when: (response) => response.alertResource === 'memorystore',
+        type: 'input',
+        name: 'instanceId',
+        message: 'Please provide the "instance id"',
+        validate: required,
+      },
+      {
+        when: (response) => response.alertResource === 'pub_sub',
+        type: 'input',
+        name: 'subscriptionId',
+        message: 'Please provide the "subscription id"',
+        validate: required,
+      },
       {
         when: (response) => response.monitoringResource === 'uptime-checks',
         type: 'input',
@@ -50,7 +100,6 @@ module.exports = class extends BaseGenerator {
         type: 'list',
         name: 'sli',
         message: 'Please select the SLI',
-        default: 'availability',
         choices: ['availability', 'error-rate', 'latency'],
       },
       {
@@ -67,11 +116,7 @@ module.exports = class extends BaseGenerator {
         name: 'info',
         message: 'WARNING: Make sure that an uptime check has been created before applying availability SLI',
       },
-    ];
-
-    return this.prompt(prompts).then((props) => {
-      this.answers = props;
-    });
+    ]);
   }
 
   async writing() {
@@ -84,20 +129,18 @@ module.exports = class extends BaseGenerator {
       burnRateAlerts,
     } = this.answers;
 
-    const serviceFolderName = serviceName.replace(/ /g, '-').toLowerCase();
-    const serviceDir = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource, serviceFolderName);
-    const uptimeDirPath = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource);
+    const resourcePath = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource);
 
     if (monitoringResource === 'uptime-checks') {
-      if (!fs.existsSync(uptimeDirPath)) {
-        fs.mkdirSync(uptimeDirPath, { recursive: true });
+      if (!fs.existsSync(resourcePath)) {
+        fs.mkdirSync(resourcePath, { recursive: true });
       }
 
-      const uptimeYamlFile = `${uptimeDirPath}/uptime-checks.yaml`;
+      const uptimeYamlFile = `${resourcePath}/uptime-checks.yaml`;
       if (!fs.existsSync(uptimeYamlFile)) {
         this.copyDir(
           'uptime-checks',
-          uptimeDirPath,
+          resourcePath,
           {
             ...this.answers,
             serviceName,
@@ -111,6 +154,8 @@ module.exports = class extends BaseGenerator {
     }
 
     if (monitoringResource === 'slos') {
+      const serviceFolderName = serviceName.replace(/ /g, '-').toLowerCase();
+      const serviceDir = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource, serviceFolderName);
       const fileContainsFilter = (fileName, str) => {
         const contents = fs.readFileSync(fileName, 'utf-8');
         const result = contents.includes(str);
@@ -167,6 +212,26 @@ module.exports = class extends BaseGenerator {
         await handleSlosFile(this.answers, sloYamlFile);
       }
     }
+
+    if (monitoringResource === 'alerts') {
+      const yamlPath = `${resourcePath}/alerts.yaml`;
+      const terraPath = `${resourcePath}/terragrunt.hcl`;
+      if (!fs.existsSync(resourcePath)) fs.mkdirSync(resourcePath, { recursive: true });
+      if (!fs.existsSync(yamlPath)) fs.writeFileSync(yamlPath, '');
+
+      if (!fs.existsSync(terraPath)) {
+        this.fs.copyTpl(
+          this.templatePath('alerts/terragrunt.hcl'),
+          this.destinationPath(terraPath),
+          this.answers,
+        );
+      }
+
+      const oldYaml = yaml.load(fs.readFileSync(yamlPath, 'utf8')) || [];
+      const newYaml = await handleAlerts(oldYaml, alertTemplates, this.answers);
+
+      fs.writeFileSync(yamlPath, yaml.dump(newYaml));
+    }
   }
 
   end() {

package/generators/resources/monitoring/templates/alerts/alerts.yaml

@@ -0,0 +1,257 @@
+cloud_run:
+  error_count:
+    display_name: "[P3] <%-systemName%>.<%-serviceName%> | 5xx Error Request Count above 1"
+    conditions:
+      - display_name: Cloud Run Anthos - 5xx error Request Count above 1
+        condition_threshold:
+          filter: |
+            resource.type="knative_revision"
+            resource.labels.service_name="<%-serviceName%>"
+            metric.type="knative.dev/serving/revision/request_count"
+            metric.labels.response_code_class="5xx"
+          threshold_value: 1
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_SUM
+              group_by_fields:
+                - metric.label.response_code_class
+              per_series_aligner: ALIGN_DELTA
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  error_rate:
+    display_name: "[P3] <%-systemName%>.<%-serviceName%> | High 5xx Error Rate"
+    conditions:
+      - display_name: Cloud Run Anthos - 3% of all requests during 10min are 5xx
+        condition_monitoring_query_language:
+          query: |
+            fetch knative_revision::knative.dev/serving/revision/request_count
+              | filter service_name = "store-data-resolver"
+              | align int_mean_aligner(10m)
+              | group_by [],    sum(if(metric.response_code_class == '5xx', val(), 0)) / sum(val())
+              | condition val() > 0.03
+              | every 10m
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  request_latency:
+    display_name: "[P3] <%-systemName%>.<%-serviceName%> | High Request Latency"
+    conditions:
+      - display_name: Cloud Run Anthos - Response Time (95%) above 1s for 5 min
+        condition_threshold:
+          filter: |
+            resource.type="knative_revision"
+            resource.labels.service_name="<%-serviceName%>"
+            metric.type="knative.dev/serving/revision/request_latencies"
+          threshold_value: 1000
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_PERCENTILE_95
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+cloud_scheduler:
+  failed_job:
+    display_name: "[P4] <%-systemName%> - Cloud Scheduler | <%-jobId%> - Job Failed"
+    conditions:
+      - display_name: Cloud Scheduler Job - Log entries with SEVERITY=Error exceed threshold
+        condition_threshold:
+          filter: |
+            resource.type="cloud_scheduler_job"
+            resource.labels.job_id="<%-jobId%>"
+            metric.type="logging.googleapis.com/log_entry_count"
+            metric.labels.severity="ERROR"
+          threshold_value: 1
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_COUNT
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+cloud_sql:
+  cpu_over_65:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - CPU over 65%"
+    conditions:
+      - display_name: Cloud SQL Database - CPU utilization above 65% over 5 min
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_database"
+            resource.labels.database_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/cpu/utilization"
+          threshold_value: 0.65
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  cpu_over_85:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - CPU over 85%"
+    conditions:
+      - display_name: "Cloud SQL Database - CPU-usage above 85% over 1 min"
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_database"
+            resource.labels.database_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/cpu/utilization"
+          threshold_value: 0.85
+          duration: 60s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  cpu_over_90:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - CPU over 90%"
+    conditions:
+      - display_name: Cloud SQL Database - CPU-usage above 90%
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_database"
+            resource.labels.database_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/cpu/utilization"
+          threshold_value: 0.9
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_50:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - Memory over 50%"
+    conditions:
+    - display_name: Cloud SQL Database - Memory utilization above 50% over 5 min
+      condition_threshold:
+        filter: |
+          resource.type="cloudsql_database"
+          resource.labels.database_id="<%-databaseId%>"
+          metric.type="cloudsql.googleapis.com/database/memory/utilization"
+        threshold_value: 50
+        duration: 300s
+        aggregations:
+        - alignment_period: 60s
+          cross_series_reducer: REDUCE_NONE
+          per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_75:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - Memory over 75%"
+    conditions:
+    - display_name: Cloud SQL Database - Memory utilization above 75% over 5 min
+      condition_threshold:
+        filter: |
+          resource.type="cloudsql_database"
+          resource.labels.database_id="<%-databaseId%>"
+          metric.type="cloudsql.googleapis.com/database/memory/utilization"
+        threshold_value: 75
+        duration: 300s
+        aggregations:
+        - alignment_period: 60s
+          cross_series_reducer: REDUCE_NONE
+          per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_90:
+    display_name: "[P3] <%-systemName%> - CloudSQL | <%-databaseId%> - Memory over 90%"
+    conditions:
+    - display_name: Cloud SQL Database - Memory utilization above 90%
+      condition_threshold:
+        filter: |
+          resource.type="cloudsql_database"
+          resource.labels.database_id="<%-databaseId%>"
+          metric.type="cloudsql.googleapis.com/database/memory/utilization"
+        threshold_value: 90
+        duration: 60s
+        aggregations:
+        - alignment_period: 60s
+          cross_series_reducer: REDUCE_NONE
+          per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  query_over_1s:
+    display_name: "[P4] <%-systemName%> - CloudSQL | <%-databaseId%> - Query resolve time"
+    conditions:
+      - display_name: Cloud SQL Instance Database - Per query execution times above 1000 ms
+        condition_threshold:
+          filter: |
+            resource.type="cloudsql_instance_database"
+            resource.labels.resource_id="<%-databaseId%>"
+            metric.type="cloudsql.googleapis.com/database/postgresql/insights/perquery/execution_time"
+          threshold_value: 1000000
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_DELTA
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+memorystore:
+  memory_over_50:
+    display_name: "[P4] <%-systemName%> - Memorystore | <%-instanceId%> - Memory over 50%"
+    conditions:
+      - display_name: Memorystore Redis Instance - Memory Usage above 50% over 5 min
+        condition_threshold:
+          filter: |
+            resource.type="redis_instance"
+            resource.labels.instance_id="<%-instanceId%>"
+            metric.type="redis.googleapis.com/stats/memory/usage_ratio"
+          threshold_value: 0.5
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_75:
+    display_name: "[P4] <%-systemName%> - Memorystore | <%-instanceId%> - Memory over 75%"
+    conditions:
+      - display_name: Memorystore Redis Instance - Memory Usage above 75% for 5min
+        condition_threshold:
+          filter: |
+            resource.type="redis_instance"
+            resource.labels.instance_id="<%-instanceId%>"
+            metric.type="redis.googleapis.com/stats/memory/usage_ratio"
+          threshold_value: 0.75
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+  memory_over_90:
+    display_name: "[P2] <%-systemName%> - Memorystore | <%-instanceId%> - Memory over 90%"
+    conditions:
+      - display_name: Memorystore Redis Instance - Memory Usage above 90%
+        condition_threshold:
+          filter: |
+            resource.type="redis_instance"
+            resource.labels.instance_id="<%-instanceId%>"
+            metric.type="redis.googleapis.com/stats/memory/usage_ratio"
+          threshold_value: 0.90
+          duration: 60s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MAX
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>
+pub_sub:
+  unacknowledged_messages:
+    display_name: "[P4] <%-systemName%> - Pub/Sub | <%-subscriptionId%> - Undelivered message(s)"
+    conditions:
+      - display_name: Cloud Pub/Sub Subscription - Undelivered messages above 1 for 5 min
+        condition_threshold:
+          filter: |
+            resource.type="pubsub_subscription"
+            resource.labels.subscription_id="<%-subscriptionId%>"
+            metric.type="pubsub.googleapis.com/subscription/num_undelivered_messages"
+          threshold_value: 1
+          duration: 300s
+          aggregations:
+            - alignment_period: 60s
+              cross_series_reducer: REDUCE_NONE
+              per_series_aligner: ALIGN_MEAN
+    documentation:
+      content: <% if (runbookLink) { %>[Runbook](<%-runbookLink%>)<%} else { %> <% } %>

package/generators/resources/monitoring/templates/alerts/terragrunt.hcl

@@ -0,0 +1,33 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/extenda/tf-module-gcp-alert-policy//?ref=v0.1.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+dependency "notification_channels" {
+  config_path  = "../notification-channels"
+  mock_outputs = {
+    notification_channels = ["dummy-channel"]
+  }
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+  common_vars  = read_terragrunt_config(find_in_parent_folders("common.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = {
+  monitoring_project_id   = local.project_vars.locals.monitoring_project_id,
+  notification_channels   = dependency.notification_channels.outputs.notification_channels,
+  policies                = yamldecode(file("${get_terragrunt_dir()}/alerts.yaml")),
+  user_labels             = {
+    cc   = local.common_vars.locals.cost_center
+    clan = local.common_vars.locals.clan_name
+  },
+}

package/generators/resources/monitoring/templates/uptime-checks/uptime-checks.yaml

@@ -1,3 +1,3 @@
-- service_name: "<%-systemName%>.<%-serviceName%>"
-  hostname: "<%-hostname%>"
-  path: "<%-path%>"
+- service_name: <%-systemName%>.<%-serviceName%>
+  hostname: <%-hostname%>
+  path: <%-path%>

package/generators/resources/monitoring/validate.js

@@ -15,4 +15,11 @@ helper.validSystemName = (input) => {
   return 'System name must be 3 characters';
 };
 
+helper.validUrl = (input) => {
+  // eslint-disable-next-line no-useless-escape
+  const regex = new RegExp(/^https:\/\/[a-zA-Z]*.[a-zA-Z]*.[a-zA-Z]*\/[a-zA-Z\/+_-]*.$/g);
+  if (regex.test(input) || input === '') return true;
+  return 'Enter a valid URL';
+};
+
 module.exports = helper;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.73.0",
+  "version": "0.74.0",
   "description": "Infrastructure as code generator for GCP.",
   "main": "src/cli.js",
   "bin": {