@hiiretail/gcp-infra-cli 0.71.0 → 0.72.0

package/generators/init/clan-infra/templates/env/project.hcl

@@ -5,7 +5,8 @@ locals {
   project_id   = "<%-clanProject%>"
   project_env  = "<%-env%>"
 
-  project          = local.project_id
-  network          = "tribe-network"
-  tribe_project_id = "<%-tribeProject%>"
+  project               = local.project_id
+  network               = "tribe-network"
+  tribe_project_id      = "<%-tribeProject%>"
+  monitoring_project_id = "<%-tribeProject%>" #  possibly will be changed to hiiretail-monitoring-prod-6500 later
 }

package/generators/init/clan-infra/templates/infra/.terraform-version

@@ -1 +1 @@
-1.0.7
+1.2.7

package/generators/init/clan-infra/templates/infra/.terragrunt-version

@@ -1 +1 @@
-0.31.8
+0.38.0

package/generators/organization/clan-project/index.js

@@ -123,7 +123,7 @@ module.exports = class extends BaseGenerator {
     this.log(`
 ${chalk.green('Your clan projects have now been created. To finalize your configuration, please continue with manual editing of the generated files.')}
 ${chalk.green('1.')} Add clan members and groups
-   \u2192 ${chalk.cyan(path.join(clanDir, 'project.yaml'))}
+   \u2192 ${chalk.cyan(path.join(clanDir, 'clan.yaml'))}
 ${chalk.green('2.')} Configure APIs, service accounts and repositories
    \u2192 ${chalk.cyan(path.join(clanDir, 'prod', 'project.yaml'))}
    \u2192 ${chalk.cyan(path.join(clanDir, 'staging', 'project.yaml'))}

package/generators/organization/clan-project/templates/clan/clan.yaml

@@ -8,8 +8,10 @@
 # members:
 #   groups: []
 #    users:
-#       - alice@extendaretail.com
-#       - bob@extendaretail.com
+#       - name: Alice Test
+#         email: alice@extendaretail.com
+#       - name: Bob Test
+#         email: bob@extendaretail.com
 ###
 ---
 common-infra-repo: <%-commonInfraRepo%>

package/generators/organization/clan-project/templates/clan-project/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v1.0.6"
+  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v1.0.8"
 }
 
 dependency "parent_folder" {

package/generators/resources/monitoring/append.js

@@ -0,0 +1,94 @@
+const fs = require('fs');
+const yaml = require('js-yaml');
+
+const appendIncludeConfigSlo = async (fileContent, originalContentYaml, slosFilePath, inputs) => {
+  if (fileContent !== null && fileContent !== '') {
+    const configArray = Object.values(originalContentYaml);
+    const yamlPullArray = yaml.dump(configArray);
+    fs.writeFileSync(slosFilePath, `${yamlPullArray}`);
+  }
+
+  const newPullArray = [];
+
+  if (inputs.sli === 'availability') {
+    newPullArray.push(
+      {
+        display_name: 'Month - Availability',
+        slo_id: 'month-availability',
+        goal: 0.998,
+        calendar_period: 'MONTH',
+        type: 'windows_based_sli',
+        method: 'boolean_filter',
+        window_period: '60s',
+      },
+    );
+  }
+  if (inputs.sli === 'error-rate') {
+    newPullArray.push(
+      {
+        display_name: 'Month - Error rate',
+        slo_id: 'month-error-rate',
+        goal: 0.999,
+        calendar_period: 'MONTH',
+        type: 'request_based_sli',
+        method: 'good_total_ratio',
+        bad_service_filter:
+`metric.type="knative.dev/serving/revision/request_count"
+resource.type="knative_revision"
+metric.labels.response_code_class="5xx"
+resource.labels.service_name="${inputs.serviceName}"`,
+        total_service_filter:
+`metric.type="knative.dev/serving/revision/request_count"
+resource.type="knative_revision"
+resource.labels.service_name=${inputs.serviceName}"`,
+      },
+    );
+  }
+  if (inputs.sli === 'latency') {
+    newPullArray.push(
+      {
+        display_name: 'Month - Latency',
+        slo_id: 'month-latency',
+        goal: 0.95,
+        calendar_period: 'MONTH',
+        type: 'request_based_sli',
+        method: 'distribution_cut',
+        metric_filter:
+`metric.type="knative.dev/serving/revision/request_latencies"
+resource.type="knative_revision"
+resource.labels.service_name="${inputs.serviceName}"`,
+        range_min: 0,
+        range_max: 100,
+      },
+    );
+  }
+
+  const finalYamlPullArray = yaml.dump(newPullArray);
+  fs.appendFileSync(slosFilePath, finalYamlPullArray);
+};
+
+const appendIncludeConfigUptime = async (fileContent, uptimeContentYml, uptimeFilePath, inputs) => {
+  if (fileContent !== null && fileContent !== '') {
+    const configArray = Object.values(uptimeContentYml);
+    const yamlPullArray = yaml.dump(configArray);
+    fs.writeFileSync(uptimeFilePath, `${yamlPullArray}`);
+  }
+
+  const newPullArray = [];
+
+  newPullArray.push(
+    {
+      service_name: `${inputs.systemName}.${inputs.serviceName}`,
+      hostname: inputs.hostname,
+      path: inputs.path,
+    },
+  );
+
+  const finalYamlPullArray = yaml.dump(newPullArray);
+  fs.appendFileSync(uptimeFilePath, finalYamlPullArray);
+};
+
+module.exports = {
+  appendIncludeConfigSlo,
+  appendIncludeConfigUptime,
+};

package/generators/resources/monitoring/generator.json

@@ -1,4 +1,4 @@
 {
   "name": "Monitoring",
-  "description": "Create monitoring dashboards and alers"
+  "description": "Create monitoring resources"
 }

package/generators/resources/monitoring/handle-slos.js

@@ -0,0 +1,26 @@
+const fs = require('fs');
+const yaml = require('js-yaml');
+const { appendIncludeConfigSlo } = require('./append');
+
+const handleSlosFile = async (answers, slosFilePath) => {
+  const {
+    serviceName,
+    sli,
+    systemName,
+  } = answers;
+
+  const sloFileContent = fs.readFileSync(slosFilePath, 'utf8');
+
+  const inputs = {
+    ...this.answers,
+    serviceName,
+    sli,
+    systemName,
+  };
+
+  const originalContentYaml = yaml.load(sloFileContent);
+  const fileContent = sloFileContent;
+  await appendIncludeConfigSlo(fileContent, originalContentYaml, slosFilePath, inputs);
+};
+
+module.exports = handleSlosFile;

package/generators/resources/monitoring/handle-uptime.js

@@ -0,0 +1,28 @@
+const fs = require('fs');
+const yaml = require('js-yaml');
+const { appendIncludeConfigUptime } = require('./append');
+
+const handleUptimeFile = async (answers, uptimeFilePath) => {
+  const {
+    serviceName,
+    hostname,
+    path,
+    systemName,
+  } = answers;
+
+  const uptimeFileContent = fs.readFileSync(uptimeFilePath, 'utf8');
+
+  const inputs = {
+    ...this.answers,
+    serviceName,
+    hostname,
+    path,
+    systemName,
+  };
+
+  const originalContentYaml = yaml.load(uptimeFileContent);
+  const fileContent = uptimeFileContent;
+  await appendIncludeConfigUptime(fileContent, originalContentYaml, uptimeFilePath, inputs);
+};
+
+module.exports = handleUptimeFile;

package/generators/resources/monitoring/index.js

@@ -1,6 +1,11 @@
 const path = require('path');
 const chalk = require('chalk');
+const fs = require('fs');
 const BaseGenerator = require('../../../src/BaseGenerator');
+const { required } = require('../../../src/validators');
+const helper = require('./validate');
+const handleSlosFile = require('./handle-slos');
+const handleUptimeFile = require('./handle-uptime');
 
 module.exports = class extends BaseGenerator {
   prompting() {
@@ -8,9 +13,51 @@ module.exports = class extends BaseGenerator {
       {
         type: 'list',
         name: 'monitoringResource',
-        message: 'Select the resource that you want to monitor',
-        default: 'dataflow',
-        choices: ['cloudrun', 'cloudfunction', 'dataflow'],
+        message: 'Select the resource you want to create',
+        default: 'uptime-checks',
+        choices: ['uptime-checks', 'slos'],
+      },
+      {
+        when: (response) => response.monitoringResource === 'uptime-checks' || 'slos',
+        type: 'input',
+        name: 'systemName',
+        message: 'Please provide three-letter system name as defined in Styra',
+        validate: required && helper.validSystemName,
+      },
+      {
+        when: (response) => response.monitoringResource === 'uptime-checks' || 'slos',
+        type: 'input',
+        name: 'serviceName',
+        message: 'Please provide the namespace where the service resides',
+        validate: required,
+      },
+      {
+        when: (response) => response.monitoringResource === 'uptime-checks',
+        type: 'input',
+        name: 'hostname',
+        message: 'Please provide the base hostname of the service (example: my-service.retailsvc.com)',
+        validate: required && helper.validHostname,
+      },
+      {
+        when: (response) => response.monitoringResource === 'uptime-checks',
+        type: 'input',
+        name: 'path',
+        message: 'Please provide the path to the page to run the check against. (example: /health)',
+        validate: required,
+      },
+      {
+        when: (response) => response.monitoringResource === 'slos',
+        type: 'list',
+        name: 'sli',
+        message: 'Please select the SLI',
+        default: 'availability',
+        choices: ['availability', 'error-rate', 'latency'],
+      },
+      {
+        when: (response) => response.monitoringResource === 'slos' && response.sli === 'availability',
+        type: 'confirm',
+        name: 'info',
+        message: 'WARNING: Make sure that an uptime check has been created before applying availability SLI',
       },
     ];
 
@@ -19,37 +66,102 @@ module.exports = class extends BaseGenerator {
     });
   }
 
-  writing() {
+  async writing() {
     const {
       monitoringResource,
+      serviceName,
+      hostname,
+      sli,
+      systemName,
     } = this.answers;
 
-    ['prod', 'staging'].forEach((env) => {
-      this.copyDir(
-        path.join(monitoringResource),
-        path.join('infra', env, 'monitoring', monitoringResource),
-        {
-          ...this.answers,
-          env,
-        },
-      );
-    });
+    const serviceFolderName = serviceName.replace(/ /g, '-').toLowerCase();
+    const serviceDir = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource, serviceFolderName);
+    const uptimeDirPath = path.join(process.cwd(), 'infra', 'prod', 'monitoring', monitoringResource);
 
-    ['prod', 'staging'].forEach((env) => {
-      this.copyDir(
-        'notification-channels',
-        path.join('infra', env, 'monitoring', 'notification-channels'),
-        {
-          ...this.answers,
-          env,
-        },
-      );
-    });
+    if (monitoringResource === 'uptime-checks') {
+      if (!fs.existsSync(uptimeDirPath)) {
+        fs.mkdirSync(uptimeDirPath, { recursive: true });
+      }
+
+      const uptimeYamlFile = `${uptimeDirPath}/uptime-checks.yaml`;
+      if (!fs.existsSync(uptimeYamlFile)) {
+        this.copyDir(
+          'uptime-checks',
+          uptimeDirPath,
+          {
+            ...this.answers,
+            serviceName,
+            hostname,
+            systemName,
+          },
+        );
+      } else {
+        await handleUptimeFile(this.answers, uptimeYamlFile);
+      }
+    }
+
+    if (monitoringResource === 'slos') {
+      const fileContainsFilter = (fileName, str) => {
+        const contents = fs.readFileSync(fileName, 'utf-8');
+        const result = contents.includes(str);
+        return result;
+      };
+
+      if (!fs.existsSync(serviceDir)) {
+        fs.mkdirSync(serviceDir, { recursive: true });
+      }
+
+      if (fs.existsSync(`${serviceDir}/terragrunt.hcl`)) {
+        if (fileContainsFilter(`${serviceDir}/terragrunt.hcl`, 'metric_filter') === false) {
+          this.fs.copyTpl(
+            this.templatePath('slos/terragrunt.hcl'),
+            this.destinationPath(`${serviceDir}/terragrunt.hcl`),
+            {
+              ...this.answers,
+              monitoringResource,
+              serviceName,
+              systemName,
+            },
+          );
+        }
+      } else {
+        this.fs.copyTpl(
+          this.templatePath('slos/terragrunt.hcl'),
+          this.destinationPath(`${serviceDir}/terragrunt.hcl`),
+          {
+            ...this.answers,
+            monitoringResource,
+            serviceName,
+            systemName,
+          },
+        );
+      }
+
+      const sloYamlFile = `${serviceDir}/slos.yaml`;
+      if (!fs.existsSync(sloYamlFile)) {
+        this.fs.copyTpl(
+          this.templatePath('slos/slos.yaml'),
+          this.destinationPath(sloYamlFile),
+          {
+            ...this.answers,
+            monitoringResource,
+            serviceName,
+            systemName,
+            sli,
+          },
+        );
+      } else {
+        await handleSlosFile(this.answers, sloYamlFile);
+      }
+    }
   }
 
   end() {
     this.log(`
 ${chalk.green('Your Monitoring resources have now been created.')}
-${chalk.green('1.')} Push this change in a feature branch and open a pull request.`);
+${chalk.green('1.')} To finalize your configuration, please continue with manual editing of the generated files.
+${chalk.green('2.')} Push the changes in a feature branch and open a pull request.
+`);
   }
 };

package/generators/resources/monitoring/templates/notification-channels/terragrunt.hcl

@@ -18,5 +18,7 @@ locals {
 inputs = merge(local.project_vars.locals, local.common_vars.locals,
   {
     clan_project_id  = local.project_vars.locals.project_id
+    # Use var below if we decide to go with hiiretail-monitoring-prod project
+    #tribe_project_id = local.common_vars.locals.monitoring_project_id
   }
 )

package/generators/resources/monitoring/templates/slos/slos.yaml

@@ -0,0 +1,32 @@
+<% if (sli === 'latency') { %>- display_name: Month - Latency
+  slo_id: month-latency
+  goal: 0.95
+  calendar_period: MONTH
+  type: request_based_sli
+  method: distribution_cut
+  metric_filter: |-
+    metric.type="knative.dev/serving/revision/request_latencies"
+    resource.type="knative_revision"
+    resource.labels.service_name="<%-serviceName%>"
+  range_min: 0
+  range_max: 100<% } %><% if (sli === 'availability') { %>- display_name: Month - Availability
+  slo_id: month-availability
+  goal: 0.998
+  calendar_period: MONTH
+  type: windows_based_sli
+  method: boolean_filter
+  window_period: 60s<% } %><% if (sli === 'error-rate') { %>- display_name: Month - Error rate
+  slo_id: month-error-rate
+  goal: 0.999
+  calendar_period: MONTH
+  type: request_based_sli
+  method: good_total_ratio
+  bad_service_filter: |-
+    metric.type="knative.dev/serving/revision/request_count"
+    resource.type="knative_revision"
+    metric.labels.response_code_class="5xx"
+    resource.labels.service_name="<%-serviceName%>"
+  total_service_filter: |-
+    metric.type="knative.dev/serving/revision/request_count"
+    resource.type="knative_revision"
+    resource.labels.service_name="<%-serviceName%>"<% } %>

package/generators/resources/monitoring/templates/slos/terragrunt.hcl

@@ -0,0 +1,36 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/extenda/tf-module-gcp-slo//?ref=v0.1.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+dependency "uptimecheck_id" {
+  config_path  = "../../uptime-checks"
+  mock_outputs = {
+    uptime_check_ids = ["dummy-id"]
+  }
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge(
+  local.project_vars.locals,
+  {
+    service_name          = "<%-systemName%>.<%-serviceName%>"
+    slos                  = yamldecode(file("${get_terragrunt_dir()}/slos.yaml")),
+    <% if (sli === 'availability') { %>
+    metric_filter = {
+      "metric.type"            = "monitoring.googleapis.com/uptime_check/check_passed"
+      "resource.type"          = "uptime_url"
+      "metric.labels.check_id" = dependency.uptimecheck_id.outputs.uptime_check_ids["<%-systemName%>.<%-serviceName%>"]
+    }<% } %>
+  }
+)

package/generators/resources/monitoring/templates/uptime-checks/terragrunt.hcl

@@ -0,0 +1,36 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/extenda/tf-module-gcp-uptime-check//?ref=v0.1.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+dependency "notification_channels" {
+  config_path  = "../notification-channels"
+  mock_outputs = {
+    notification_channels = ["dummy-channel"]
+  }
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+  common_vars  = read_terragrunt_config(find_in_parent_folders("common.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge(
+  local.project_vars.locals,
+  local.common_vars.locals,
+  {
+    notification_channels = dependency.notification_channels.outputs.notification_channels
+    uptime_checks         = yamldecode(file("${get_terragrunt_dir()}/uptime-checks.yaml")),
+    labels                = {
+      clan = local.common_vars.locals.clan_name
+      cc   = local.common_vars.locals.cost_center
+    }
+  }
+)

package/generators/resources/monitoring/templates/uptime-checks/uptime-checks.yaml

@@ -0,0 +1,3 @@
+- service_name: "<%-systemName%>.<%-serviceName%>"
+  hostname: "<%-hostname%>"
+  path: "<%-path%>"

package/generators/resources/monitoring/validate.js

@@ -0,0 +1,18 @@
+const helper = {};
+
+helper.validHostname = (input) => {
+  const regex = new RegExp(/^(?:[a-z-]+\.){1,3}[a-z-]+$/g);
+  if (input.match(regex)) {
+    return true;
+  }
+  return 'Hostname must not include path to the page to run the check against or spaces';
+};
+
+helper.validSystemName = (input) => {
+  if (input.replace(/\s/g, '').length === 3) {
+    return true;
+  }
+  return 'System name must be 3 characters';
+};
+
+module.exports = helper;

package/generators/tribe-resources/tribe-project/templates/project/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v1.0.6"
+  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v1.0.8"
 }
 
 dependency "tribe_folder" {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.71.0",
+  "version": "0.72.0",
   "description": "Infrastructure as code generator for GCP.",
   "main": "src/cli.js",
   "bin": {