@hiiretail/gcp-infra-cli 0.87.4 → 0.88.0

package/generators/clan-resources/clan-project/generator.json

@@ -0,0 +1,4 @@
+{
+  "name": "Clan project with GKE Autopilot",
+  "description": "Create resources for clan project with GKE cluster"
+}

package/generators/clan-resources/clan-project/index.js

@@ -0,0 +1,135 @@
+const path = require('path');
+const fs = require('fs');
+const chalk = require('chalk');
+const BaseGenerator = require('../../../src/BaseGenerator');
+const { required } = require('../../../src/validators');
+
+module.exports = class extends BaseGenerator {
+  constructor(args, opts) {
+    super(args, opts);
+
+    // Output directory relative to the tf-infra-gcp directory.
+    this.destinationRoot('organization');
+  }
+
+  prompting() {
+    const prompts = [
+      {
+        type: 'list',
+        name: 'domain',
+        default: 'extendaretail-com',
+        message: 'Choose GCP Organization',
+        store: true,
+        choices: this.listSubDirectories(this.destinationPath()),
+        validate: required,
+      },
+      {
+        type: 'list',
+        name: 'department',
+        message: 'Choose department',
+        store: true,
+        choices: (answers) => this.listSubDirectories(
+          path.join(this.destinationPath(), answers.domain, 'departments'),
+        ),
+        validate: required,
+      },
+      {
+        type: 'list',
+        name: 'tribe',
+        message: 'Choose tribe',
+        store: true,
+        choices: (answers) => this.listSubDirectories(
+          path.join(this.destinationPath(), answers.domain, 'departments', answers.department),
+        ),
+        validate: required,
+      },
+      {
+        type: 'list',
+        name: 'clan',
+        message: 'Choose clan',
+        store: true,
+        choices: (answers) => this.listSubDirectories(
+          path.join(this.destinationPath(), answers.domain, 'departments', answers.department, answers.tribe, 'clans'),
+        ),
+        validate: required,
+      },
+      {
+        type: 'list',
+        name: 'migration',
+        message: 'Is the project in the process of migration to Autopilot?',
+        default: 'yes',
+        choices: ['yes', 'no'],
+      },
+    ];
+
+    return this.prompt(prompts).then((props) => {
+      this.answers = props;
+    });
+  }
+
+  writing() {
+    const {
+      domain,
+      department,
+      tribe,
+      clan,
+      migration,
+    } = this.answers;
+
+    ['prod', 'staging'].forEach((env) => {
+      this.copyDir(
+        'clan-resources',
+        path.join(domain, 'departments', department, tribe, 'clans', clan, env, 'resources'),
+        {
+          ...this.answers,
+          env,
+        },
+      );
+    });
+
+    if (migration === 'yes') {
+      ['prod', 'staging'].forEach((env) => {
+        const terragruntFilePath = path.join(domain, 'departments', department, tribe, 'clans', clan, env, 'terragrunt.hcl');
+
+        fs.readFile(terragruntFilePath, 'utf8', (err, data) => {
+          if (err) {
+            return;
+          }
+
+          const modifiedData = data.replace(
+            /source\s*=\s*"git::https:\/\/github\.com\/extenda\/tf-module-gcp-project\/\/\?ref=.+"/,
+            'source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=migration"',
+          );
+
+          fs.writeFile(terragruntFilePath, modifiedData, 'utf8', (error) => {
+            if (error) {
+              console.error(`Error writing file ${terragruntFilePath}: ${error}`);
+            } else {
+              console.log(`Updated file ${terragruntFilePath}`);
+            }
+          });
+        });
+      });
+    }
+  }
+
+  end() {
+    const {
+      domain,
+      department,
+      tribe,
+      clan,
+    } = this.answers;
+    const clanDir = path.join(domain, 'departments', department, tribe, 'clans', clan);
+    this.log(`
+${chalk.green(`Your clan resources have now been created. To finalize your configuration, please continue
+with manual editing of the generated files.`)}
+${chalk.green('1.')} Review the configurations
+   \u2192 ${chalk.cyan(path.join(clanDir, 'prod', 'network', 'spec.hcl'))}
+   \u2192 ${chalk.cyan(path.join(clanDir, 'prod', 'gke', 'spec.hcl'))}
+   \u2192 ${chalk.cyan(path.join(clanDir, 'staging', 'network', 'spec.hcl'))}
+   \u2192 ${chalk.cyan(path.join(clanDir, 'staging', 'gke', 'spec.hcl'))}
+${chalk.green('2.')} Push this change in a feature branch and open a pull request.
+`);
+  }
+};

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/cloud-nat/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-cloud-nat//?ref=v2.0.0"
+  source = "git::https://github.com/terraform-google-modules/terraform-google-cloud-nat//?ref=v2.2.2"
 }
 
 # Include all settings from the root terragrunt.hcl file
@@ -29,18 +29,24 @@ dependency "router" {
 dependency "static_ip" {
   config_path = "../cloud-nat-static-ip"
   mock_outputs = {
-    self_links = ["https://www.googleapis.com/compute/v1/projects/project-id/regions/europe-west1/addresses/ip"]
+    names = "ip-name"
   }
 }
 
+locals {
+  clan_name = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
+}
+
 # These are the variables we have to pass in to use the module specified in the terragrunt configuration above
 inputs = merge(
   {
-    name       = "<%- tribe %>-cloud-nat"
+    name       = "${local.clan_name}-cloud-nat"
     region     = "europe-west1"
     project_id = dependency.network.outputs.project_id
     network    = dependency.network.outputs.network_name
     router     = dependency.router.outputs.router.name
     nat_ips    = dependency.static_ip.outputs.self_links
+
+    enable_endpoint_independent_mapping = false
   }
 )

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/cloud-nat-static-ip/terragrunt.hcl

@@ -1,9 +1,9 @@
 terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-address//?ref=v3.0.0"
+  source = "git::https://github.com/terraform-google-modules/terraform-google-address//?ref=v3.1.2"
 }
 
 dependency "project" {
-  config_path = "../"
+  config_path = "../../"
   mock_outputs = {
     project_id = "project-dummy-id"
   }

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/cloud-router/terragrunt.hcl

@@ -1,9 +1,9 @@
 terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-cloud-router//?ref=v1.1.0"
+  source = "git::https://github.com/terraform-google-modules/terraform-google-cloud-router//?ref=v5.0.0"
 }
 
 dependency "project" {
-  config_path = "../"
+  config_path = "../../"
   mock_outputs = {
     project_id = "project-dummy-id"
   }
@@ -13,7 +13,6 @@ dependency "network" {
   config_path = "../network"
   mock_outputs = {
     network_name = "network-dummy-name"
-    subnets_names = "subnet-dummy-name"
   }
 }
 
@@ -21,9 +20,13 @@ include {
   path = find_in_parent_folders("terragrunt_root.hcl")
 }
 
+locals {
+  clan_name = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
+}
+
 inputs = merge(
   {
-    name    = "<%- tribe %>-router"
+    name    = "${local.clan_name}-router"
     region  = "europe-west1"
     project = dependency.project.outputs.project_id
     network = dependency.network.outputs.network_name

package/generators/clan-resources/clan-project/templates/clan-resources/gke/gke-resources/terragrunt.hcl

@@ -0,0 +1,40 @@
+terraform {
+  source = "git::https://github.com/extenda/tf-module-gcp-project//modules/gke-resources?ref=v2.0.0"
+}
+
+dependency "project" {
+  config_path = "../../../"
+  mock_outputs = {
+    project_id = "project-dummy-id"
+    ci_cd_service_account_email =  {
+      "ci-cd-pipeline" = "dummy"
+    }
+  }
+}
+
+dependency "gke" {
+  config_path = "../"
+  mock_outputs = {
+    endpoint       = "dummy-endpoint"
+    ca_certificate = "dummy-ca-certificate"
+   }
+}
+
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+locals {
+  clan_name    = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
+  project_yaml = yamldecode(file(find_in_parent_folders("project.yaml")))
+}
+
+inputs = merge(
+  {
+    services           = local.project_yaml.services
+    project_id         = dependency.project.outputs.project_id
+    gke_host           = dependency.gke.outputs.endpoint
+    gke_ca_certificate = dependency.gke.outputs.ca_certificate
+    cicd_service       = dependency.project.outputs.ci_cd_service_account_email["ci-cd-pipeline"]
+  }
+)

package/generators/clan-resources/clan-project/templates/clan-resources/gke/spec.hcl

@@ -0,0 +1,18 @@
+locals {
+  regional                        = true
+  region                          = "europe-west1"
+  zones                           = ["europe-west1-d", "europe-west1-c", "europe-west1-b"]
+  ip_range_pods                   = "pods-cidr"
+  ip_range_services               = "services-cidr"
+  create_service_account          = true
+  grant_registry_access           = true
+  http_load_balancing             = true
+  add_cluster_firewall_rules      = true
+  dns_cache                       = true
+  enable_vertical_pod_autoscaling = true
+  enable_private_nodes            = true
+  master_global_access_enabled    = false
+  release_channel                 = "REGULAR"
+  enable_cost_allocation          = true
+  maintenance_start_time          = "01:00" #UTC
+}

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/gke/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-kubernetes-engine//modules/beta-private-cluster?ref=v13.0.0"
+  source = "git::https://github.com/terraform-google-modules/terraform-google-kubernetes-engine//modules/beta-autopilot-private-cluster?ref=v25.0.0"
 }
 
 # Include all settings from the root terragrunt.hcl file
@@ -10,7 +10,7 @@ include {
 }
 
 dependency "project" {
-  config_path = "../"
+  config_path = "../../"
   mock_outputs = {
     project_id = "project-dummy-id"
   }
@@ -20,27 +20,32 @@ dependency "network" {
   config_path = "../network"
   mock_outputs = {
     network_name = "network-dummy-name"
-    subnets_names = "subnet-dummy-name"
   }
 }
 
 locals {
-  spec_vars = read_terragrunt_config("${get_terragrunt_dir()}/spec.hcl")
-  tribe_yaml  = yamldecode(file(find_in_parent_folders("tribe.yaml")))
+  spec_vars  = read_terragrunt_config("${get_terragrunt_dir()}/spec.hcl")
+  clan_yaml = yamldecode(file(find_in_parent_folders("clan.yaml")))
+  clan_name = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
+  env_name = basename(dirname(dirname(get_terragrunt_dir())))
 }
 
 # These are the variables we have to pass in to use the module specified in the terragrunt configuration above
 inputs = merge(
   local.spec_vars.locals,
   {
+    name                         = "${local.clan_name}-cluster-${local.env_name}"
     project_id                   = dependency.project.outputs.project_id
     network                      = dependency.network.outputs.network_name
     subnetwork                   = "k8s-subnet"
     identity_namespace           = "${dependency.project.outputs.project_id}.svc.id.goog"
     authenticator_security_group = "gke-security-groups@extendaretail.com"
-    registry_project_id          = "extenda"
+    registry_project_ids         = ["extenda"]
     cluster_resource_labels      = {
-      cc = local.tribe_yaml.costcenter
+      cc = local.clan_yaml.costcenter
     }
+
+    // Binary Auth must be Enabled manually in this console - not supported by the module yet
+    enable_binary_authorization  = true
   }
 )

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/network/firewall-rules/config/main.tf

@@ -13,3 +13,16 @@ resource "google_compute_firewall" "dataflow-workers" {
   source_tags = ["dataflow"]
   target_tags = ["dataflow"]
 }
+
+resource "google_compute_firewall" "allow_negs" {
+  name    = "allow-negs"
+  network = var.network
+  project = var.project
+
+  allow {
+    protocol = "all"
+  }
+  source_ranges = ["35.191.0.0/16", "130.211.0.0/22"]
+  priority      = 1000
+  direction     = "INGRESS"
+}

package/generators/clan-resources/clan-project/templates/clan-resources/network/firewall-rules/config/vars.tf

@@ -0,0 +1,9 @@
+variable "network" {
+  description = "The name or self_link of the network to attach this firewall to"
+  type        = string
+}
+
+variable "project" {
+  description = "The ID of the project in which the resource belongs"
+  type        = string
+}

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/network/firewall-rules/terragrunt.hcl

@@ -16,8 +16,8 @@ dependency "network" {
   }
 }
 
-dependency "tribe_project" {
-  config_path = "../../"
+dependency "clan_project" {
+  config_path = "../../../"
   mock_outputs = {
     project_id = "dummy-id"
   }
@@ -27,6 +27,6 @@ dependency "tribe_project" {
 inputs = merge(
   {
     network     = dependency.network.outputs.network_self_link
-    project     = dependency.tribe_project.outputs.project_id
+    project     = dependency.clan_project.outputs.project_id
   }
 )

package/generators/clan-resources/clan-project/templates/clan-resources/network/spec.hcl

@@ -0,0 +1,46 @@
+locals {
+  network_name    = "clan-network"
+
+  subnets = [
+    {
+      subnet_name   = "k8s-subnet"
+      subnet_ip     = "10.152.16.0/23"
+      subnet_region = "europe-west1"
+      subnet_private_access = true
+    },
+    {
+      subnet_name   = "clan-resources"
+      subnet_ip     = "10.152.20.0/22"
+      subnet_region = "europe-west1"
+      subnet_private_access = true
+    },
+    {
+      subnet_name = "vpc-connector-subnet"
+      subnet_ip     = "10.8.0.0/28"
+      subnet_region = "europe-west1"
+      description   = "The subnet must be used only for Serverless VPC connector"
+      subnet_private_access = true
+    },
+    {
+      subnet_name   = "internal-lb-subnet"
+      subnet_ip     = "10.10.0.0/23"
+      subnet_region = "europe-west1"
+      purpose       = "REGIONAL_MANAGED_PROXY"
+      role          = "ACTIVE"
+      description   = "The subnet must be used only for Serverless VPC connector"
+    }
+  ]
+
+  secondary_ranges = {
+    "k8s-subnet" = [
+      {
+        range_name    = "pods-cidr"
+        ip_cidr_range = "172.18.0.0/20"
+      },
+      {
+        range_name    = "services-cidr"
+        ip_cidr_range = "172.19.0.0/20"
+      }
+    ]
+  }
+}

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/network/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-network//?ref=v3.3.0"
+  source = "git::https://github.com/terraform-google-modules/terraform-google-network//?ref=v6.0.1"
 }
 
 # Include all settings from the root terragrunt.hcl file
@@ -10,7 +10,7 @@ include {
 }
 
 dependency "project" {
-  config_path = "../"
+  config_path = "../../"
   mock_outputs = {
     project_id = "project-dummy-id"
   }

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/private-vpc-connection/config/main.tf

@@ -2,7 +2,8 @@ resource "google_compute_global_address" "private_ip_alloc" {
   provider = google-beta
 
   project       = var.project
-  name          = "private-ip-alloc"
+  name          = "private-ip-peering"
+  description   = "VPC Private connection"
   purpose       = var.purpose
   address_type  = var.address_type
   prefix_length = var.prefix_length

package/generators/{tribe-resources/tribe-project/templates/project → clan-resources/clan-project/templates/clan-resources}/private-vpc-connection/terragrunt.hcl

@@ -3,7 +3,7 @@ terraform {
 }
 
 dependency "project" {
-  config_path = "../"
+  config_path = "../../"
   mock_outputs = {
     project_id = "project-dummy-id"
   }
@@ -21,7 +21,7 @@ include {
 }
 
 locals {
-  tribe_yaml  = yamldecode(file(find_in_parent_folders("tribe.yaml")))
+  clan_yaml = yamldecode(file(find_in_parent_folders("clan.yaml")))
 }
 
 inputs = merge(
@@ -29,7 +29,7 @@ inputs = merge(
     project        = dependency.project.outputs.project_id
     network        = dependency.network.outputs.network_self_link
     address_labels = {
-      cc = local.tribe_yaml.costcenter
+      cc = local.clan_yaml.costcenter
     }
   }
 )

package/generators/clan-resources/clan-project/templates/clan-resources/serverless-vpc-connector/terragrunt.hcl

@@ -0,0 +1,40 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/terraform-google-modules/terraform-google-network//modules/vpc-serverless-connector-beta?ref=v6.0.1"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+dependency "project" {
+  config_path = "../../"
+  mock_outputs = {
+    project_id = "project-dummy-id"
+  }
+}
+
+locals {
+  clan_name = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge(
+  {
+    project_id = dependency.project.outputs.project_id
+    vpc_connectors = [
+      {
+        name            = "${local.clan_name}-connector"
+        region          = "europe-west1"
+        subnet_name     = "vpc-connector-subnet"
+        host_project_id = dependency.project.outputs.project_id
+        machine_type    = "e2-micro"
+        min_instances   = 2
+        max_instances   = 3
+        max_throughput  = 300
+      }
+    ]
+  }
+)

package/generators/{tribe-resources → clan-resources}/command.json

@@ -1,5 +1,5 @@
 {
-  "description": "Create tribe project with GKE and related resources",
+  "description": "Create clan resources with GKE Autopilot cluster.",
   "query": "What do you want to create?",
   "require": [
     "org"

package/generators/{resources → common-resources}/command.json

@@ -1,5 +1,5 @@
 {
-  "description": "Create GCP resources.",
+  "description": "Create common clan GCP resources.",
   "query": "What do you want to create?",
   "require": [
     "git-repo",

package/generators/{resources → common-resources}/spanner/index.js

@@ -63,10 +63,7 @@ module.exports = class extends BaseGenerator {
         type: 'input',
         name: 'databaseName',
         message: 'Please provide a name for the database',
-        validate: (input) => {
-          const validation = validName(input);
-          return validation.valid ? true : validation.message;
-        },
+        validate: validName,
       },
       {
         when: (response) => response.resourceType === 'database',
@@ -79,10 +76,7 @@ module.exports = class extends BaseGenerator {
         type: 'input',
         name: 'retentionPeriod',
         message: 'Please provide version retention period. Maximum value 7d, possible values include 84000s, 1h, 2d. Leave empty for default value 1h.',
-        validate: (input) => {
-          const validation = validRetentionPeriod(input);
-          return validation.valid ? true : validation.message;
-        },
+        validate: validRetentionPeriod,
       },
     ];
 

package/generators/common-resources/spanner/validate.js

@@ -0,0 +1,24 @@
+const validName = (input) => {
+  if (input.replace(/\s/g, '').length >= 6 && input.length <= 30) {
+    return true;
+  }
+  return 'Name must be between 6 and 30 characters in length';
+};
+
+const validRetentionPeriod = (input) => {
+  const validFormats = /^(84000s|1h|2d|7d)$/;
+
+  if (input === '') {
+    return true;
+  }
+  if (validFormats.test(input)) {
+    return true;
+  }
+
+  return 'Invalid retention period format. Possible values are 84000s, 1h, 2d, or leave empty for default value 1h';
+};
+
+module.exports = {
+  validName,
+  validRetentionPeriod,
+};

package/generators/organization/clan-project/index.js

@@ -55,6 +55,12 @@ module.exports = class extends BaseGenerator {
         message: 'Name of GitHub repository for clan common infrastructure',
         default: (answers) => `${answers.tribe}-${answers.clan}-common`,
       },
+      {
+        type: 'input',
+        name: 'costCenter',
+        message: 'Please provide the Cost Center of the Clan',
+        validate: required,
+      },
     ];
 
     return this.prompt(prompts).then((props) => {

package/generators/organization/clan-project/templates/clan/clan.yaml

@@ -14,7 +14,9 @@
 #         email: bob@extendaretail.com
 ###
 ---
-common-infra-repo: <%-commonInfraRepo%>
+common-infra-repo: "<%-commonInfraRepo%>"
 members:
   groups: []
   users: []
+
+costcenter: "<%-costCenter%>"