@hiiretail/gcp-infra-generators 1.0.1 → 1.1.0

package/dist/generators/common-resources/monitoring/templates/alerts/generic-infra.yaml

@@ -19,6 +19,7 @@ cloud_function:
                 - resource.label.function_name
     documentation:
       subject: 'Function: `$${resource.label.function_name}`'
+      content: 'Review the function in Cloud Functions: https://console.cloud.google.com/run?deploymentType=function&project=<%-projectId%>'
 cloud_run:
   error_count:
     display_name: "[P3] <%-clan%> - Cloud Run | Error rate"
@@ -40,6 +41,7 @@ cloud_run:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   request_latency:
     display_name: "[P3] <%-clan%> - Cloud Run | Request Latency"
     conditions:
@@ -59,6 +61,7 @@ cloud_run:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   cpu_utilization:
     display_name: "[P3] <%-clan%> - Cloud Run | CPU Utilization"
     conditions:
@@ -72,12 +75,13 @@ cloud_run:
           duration: 300s
           aggregations:
             - alignment_period: 60s
-              per_series_aligner: ALIGN_MEAN
+              per_series_aligner: ALIGN_PERCENTILE_99
               cross_series_reducer: REDUCE_PERCENTILE_99
               group_by_fields:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   memory_utilization:
     display_name: "[P3] <%-clan%> - Cloud Run | Memory Utilization"
     conditions:
@@ -91,12 +95,13 @@ cloud_run:
           duration: 300s
           aggregations:
             - alignment_period: 60s
-              per_series_aligner: ALIGN_MEAN
+              per_series_aligner: ALIGN_PERCENTILE_99
               cross_series_reducer: REDUCE_PERCENTILE_99
               group_by_fields:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   startup_latency:
     display_name: "[P4] <%-clan%> - Cloud Run | Startup Latency"
     conditions:
@@ -116,6 +121,7 @@ cloud_run:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
 cloud_scheduler:
   failed_job:
     display_name: "[P4] <%-clan%> - Cloud Scheduler | Job Failed"
@@ -135,6 +141,7 @@ cloud_scheduler:
                 - resource.label.job_id
     documentation:
       subject: 'Job: `$${resource.label.job_id}`'
+      content: 'Review the job in Cloud Scheduler: https://console.cloud.google.com/cloudscheduler?project=<%-projectId%>'
   failed_executions:
     display_name: "[P3] <%-clan%> - Cloud Scheduler | Failed Executions"
     conditions:
@@ -155,6 +162,7 @@ cloud_scheduler:
                 - resource.label.job_id
     documentation:
       subject: 'Job: `$${resource.label.job_id}`'
+      content: 'Review the job in Cloud Scheduler: https://console.cloud.google.com/cloudscheduler?project=<%-projectId%>'
 cloud_sql:
   cpu_over_65:
     display_name: "[P3] <%-clan%> - CloudSQL | CPU over 65%"
@@ -175,6 +183,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   cpu_over_85:
     display_name: "[P3] <%-clan%> - CloudSQL | CPU over 85%"
     conditions:
@@ -194,6 +203,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   cpu_over_90:
     display_name: "[P3] <%-clan%> - CloudSQL | CPU over 90%"
     conditions:
@@ -213,6 +223,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   memory_over_90:
     display_name: "[P2] <%-clan%> - CloudSQL | Memory utilization above 90%"
     conditions:
@@ -232,6 +243,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   query_over_1s:
     display_name: "[P4] <%-clan%> - CloudSQL | Slow query"
     conditions:
@@ -251,6 +263,7 @@ cloud_sql:
                 - resource.label.resource_id
     documentation:
       subject: 'Database: `$${resource.label.resource_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   postgresql_connections:
     display_name: "[P3] <%-clan%> - CloudSQL | PostgreSQL Connections"
     conditions:
@@ -269,6 +282,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   aggregated_lock_time:
     display_name: "[P4] <%-clan%> - CloudSQL | Aggregated Lock Time"
     conditions:
@@ -287,6 +301,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
 firestore:
   request_latencies:
     display_name: "[P3] <%-clan%> - Firestore | High request latencies"
@@ -306,6 +321,7 @@ firestore:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Firestore: https://console.cloud.google.com/firestore/databases?project=<%-projectId%>'
   failed_commits:
     display_name: "[P3] <%-clan%> - Firestore | High failed commits errors"
     conditions:
@@ -324,6 +340,9 @@ firestore:
               cross_series_reducer: REDUCE_SUM
               group_by_fields:
                 - metric.label."response_code"
+    documentation:
+      subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Firestore: https://console.cloud.google.com/firestore/databases?project=<%-projectId%>'
 memorystore:
   memory_over_50:
     display_name: "[P3] <%-clan%> - Memorystore | Memory over 50%"
@@ -344,6 +363,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   memory_over_75:
     display_name: "[P3] <%-clan%> - Memorystore | Memory over 75%"
     conditions:
@@ -363,6 +383,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   memory_over_90:
     display_name: "[P2] <%-clan%> - Memorystore | Memory over 90%"
     conditions:
@@ -382,6 +403,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   cpu_utilization:
     display_name: "[P2] <%-clan%> - Memorystore | CPU Utilization"
     conditions:
@@ -401,6 +423,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   system_memory_overload_duration:
     display_name: "[P1] <%-clan%> - Memorystore | System Memory Overload Duration"
     conditions:
@@ -419,6 +442,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   calls:
     display_name: "[P3] <%-clan%> - Memorystore | Calls"
     conditions:
@@ -438,6 +462,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
 pub_sub:
   unacknowledged_messages:
     display_name: "[P3] <%-clan%> - Pub/Sub | Undelivered message(s)"
@@ -457,6 +482,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   messages_in_dlq:
     display_name: "[P3] <%-clan%> - Pub/Sub | Message(s) in DLQ"
     conditions:
@@ -477,6 +503,7 @@ pub_sub:
                 - metric.label.response_code
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`, Response code: `$${metric.label.response_code}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   latency:
     display_name: "[P3] <%-clan%> - Pub/Sub | Response latency distribution"
     conditions:
@@ -496,6 +523,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   oldest_unacked_message_age:
     display_name: "[P3] <%-clan%> - Pub/Sub | Oldest unacked message age"
     conditions:
@@ -514,6 +542,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   push_requests_ack:
     display_name: "[P3] <%-clan%> - Pub/Sub | Push requests ack"
     conditions:
@@ -533,6 +562,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   push_requests_success:
     display_name: "[P3] <%-clan%> - Pub/Sub | Push requests success"
     conditions:
@@ -552,6 +582,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   delivery_latency_health_score:
     display_name: "[P3] <%-clan%> - Pub/Sub | Delivery latency health score"
     conditions:
@@ -571,6 +602,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
 spanner:
   cpu_utilization_by_priority:
     display_name: "[P2] <%-clan%> - Spanner | CPU Utilization"
@@ -591,6 +623,7 @@ spanner:
                 - metric.label.database
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`, Database: `$${resource.label.database}`'
+      content: 'Review the instance in Spanner: https://console.cloud.google.com/spanner/instances?project=<%-projectId%>'
   api_request_error_rate:
     display_name: "[P2] <%-clan%> - Spanner | API request error rate"
     conditions:
@@ -611,6 +644,7 @@ spanner:
                 - resource.label.database
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`, Database: `$${resource.label.database}`'
+      content: 'Review the instance in Spanner: https://console.cloud.google.com/spanner/instances?project=<%-projectId%>'
   request_latencies:
     display_name: '[P3] <%-clan%> - Spanner | API transaction latency'
     conditions:
@@ -631,3 +665,4 @@ spanner:
               per_series_aligner: ALIGN_PERCENTILE_95
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`, Database: `$${resource.label.database}}`'
+      content: 'Review the instance in Spanner: https://console.cloud.google.com/spanner/instances?project=<%-projectId%>'

package/dist/generators/common-resources/pubsub/index.js

@@ -18,7 +18,7 @@ module.exports = class extends BaseGenerator {
       {
         type: 'list',
         name: 'createResource',
-        message: 'Do you want to create a new topic or a subscription?',
+        message: 'Do you want to create a new topic, subscription?',
         default: 'subscription',
         choices: ['topic', 'subscription'],
       },
@@ -168,6 +168,15 @@ module.exports = class extends BaseGenerator {
           'Please provide the audience that will be used when generating OIDC token',
         validate: required,
       },
+      {
+        when: (response) => response.createResource === 'topic',
+        type: 'list',
+        name: 'externalTopicPublishers',
+        message: 'Do you want to allow external publishers onto this topic?',
+        default: 'no',
+        choices: ['no', 'yes'],
+        validate: required,
+      },
     ];
 
     return this.prompt(prompts).then((props) => {
@@ -195,6 +204,7 @@ module.exports = class extends BaseGenerator {
       prodProjectIdConsumer,
       costCenter,
       externalSub,
+      externalTopicPublishers,
     } = this.answers;
 
     const dlqTopicName = `dlq.${getProjectId('prod').split('-prod')[0]}.common`;
@@ -224,7 +234,8 @@ module.exports = class extends BaseGenerator {
           'pubsub',
           topicName,
         );
-        ['terragrunt.hcl', 'spec.hcl'].forEach(async (file) => {
+        const files = ['terragrunt.hcl', 'spec.hcl'];
+        for (const file of files) {
           this.fs.copyTpl(
             this.templatePath(`pubsub/${file}`),
             this.destinationPath(`${topicDirPath}/${file}`),
@@ -236,7 +247,7 @@ module.exports = class extends BaseGenerator {
               dlqTopic,
             },
           );
-        });
+        }
       }
 
       let projectId = getProjectId(env);
@@ -276,7 +287,8 @@ module.exports = class extends BaseGenerator {
           );
         }
 
-        ['terragrunt.hcl', 'spec.hcl'].forEach(async (file) => {
+        const files = ['terragrunt.hcl', 'spec.hcl'];
+        for (const file of files) {
           this.fs.copyTpl(
             this.templatePath(`pubsub/${file}`),
             this.destinationPath(`${subscriptionDirPath}/${file}`),
@@ -286,7 +298,7 @@ module.exports = class extends BaseGenerator {
               existingTopic,
             },
           );
-        });
+        }
 
         await handleSubscribers(
           env,
@@ -297,11 +309,12 @@ module.exports = class extends BaseGenerator {
           dlqTopic,
         );
       }
+
       if (createResource === 'subscription' && externalSub === 'yes') {
         const externalDirPath = path.join(
           process.cwd(),
           'infra',
-          'prod',
+          env,
           'pubsub',
           existingTopic,
           `${clanName}-${env}`,
@@ -311,7 +324,8 @@ module.exports = class extends BaseGenerator {
           fs.mkdirSync(externalDirPath);
         }
 
-        ['terragrunt.hcl', 'spec.hcl'].forEach(async (file) => {
+        const files = ['terragrunt.hcl', 'spec.hcl'];
+        for (const file of files) {
           this.fs.copyTpl(
             this.templatePath(`pubsub-external/${file}`),
             this.destinationPath(`${externalDirPath}/${file}`),
@@ -326,7 +340,7 @@ module.exports = class extends BaseGenerator {
               projectId,
             },
           );
-        });
+        }
 
         const externalSubPath = `${externalDirPath}/subscribers.yaml`;
         if (env === 'staging') {
@@ -362,6 +376,28 @@ module.exports = class extends BaseGenerator {
           dlqTopic,
         );
       }
+
+      if (createResource === 'topic' && externalTopicPublishers === 'yes') {
+        const externalPublishersPath = path.join(
+          process.cwd(),
+          'infra',
+          env,
+          'pubsub',
+          topicName,
+          'external-publishers',
+        );
+
+        const files = ['terragrunt.hcl', 'bindings.yaml'];
+        for (const file of files) {
+          this.fs.copyTpl(
+            this.templatePath(`pubsub-external-publishers/${file}`),
+            this.destinationPath(`${externalPublishersPath}/${file}`),
+            {
+              ...this.answers,
+            },
+          );
+        }
+      }
     });
   }
 
@@ -370,7 +406,8 @@ module.exports = class extends BaseGenerator {
 ${chalk.green(`Your PubSub resources have now been created. To finalize your configuration, please continue
 with manual editing of the generated files.`)}
 ${chalk.green('1.')} Review created subscribers.
-${chalk.green('2.')} Push this change in a feature branch and open a pull request.
+${chalk.green('2.')} Review created external publishers.
+${chalk.green('3.')} Push this change in a feature branch and open a pull request.
 `);
   }
 };

package/dist/generators/common-resources/pubsub/templates/pubsub-external-publishers/bindings.yaml

@@ -0,0 +1,8 @@
+<% if (createResource == 'topic' && externalTopicPublishers == 'yes') { %>clans:
+# Add External serviceAccounts like:
+# clans:
+#   iam:
+#    - serviceAccount: ...
+#  store-data:
+#    - serviceAccount: ...
+<% } %>

package/dist/generators/common-resources/pubsub/templates/pubsub-external-publishers/terragrunt.hcl

@@ -0,0 +1,37 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/terraform-google-modules/terraform-google-iam.git//modules/pubsub_topics_iam?ref=v8.1.0"
+}
+
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+  bindings_raw = yamldecode(file("${get_terragrunt_dir()}/bindings.yaml"))
+  clan_members = flatten([
+    for clan, members in local.bindings_raw.clans : [
+      for m in members : (
+        can(m.serviceAccount) ? m.serviceAccount : regex_replace("^serviceAccount:", "", m)
+      )
+    ]
+  ])
+
+  iam_bindings = {
+    "roles/pubsub.publisher" = [
+      for sa in local.clan_members : "serviceAccount:${sa}"
+    ]
+  }
+}
+
+dependency "topic" {
+  config_path = "../"
+}
+
+inputs =  {
+    project = local.project_vars.locals.project_id
+    pubsub_topics = [dependency.topic.outputs.topic]
+    bindings = local.iam_bindings
+}