@hiiretail/gcp-infra-generators 1.0.0 → 1.0.2

package/dist/generators/common-resources/monitoring/templates/alerts/generic-infra.yaml

@@ -19,6 +19,7 @@ cloud_function:
                 - resource.label.function_name
     documentation:
       subject: 'Function: `$${resource.label.function_name}`'
+      content: 'Review the function in Cloud Functions: https://console.cloud.google.com/run?deploymentType=function&project=<%-projectId%>'
 cloud_run:
   error_count:
     display_name: "[P3] <%-clan%> - Cloud Run | Error rate"
@@ -40,6 +41,7 @@ cloud_run:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   request_latency:
     display_name: "[P3] <%-clan%> - Cloud Run | Request Latency"
     conditions:
@@ -59,6 +61,7 @@ cloud_run:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   cpu_utilization:
     display_name: "[P3] <%-clan%> - Cloud Run | CPU Utilization"
     conditions:
@@ -72,12 +75,13 @@ cloud_run:
           duration: 300s
           aggregations:
             - alignment_period: 60s
-              per_series_aligner: ALIGN_MEAN
+              per_series_aligner: ALIGN_PERCENTILE_99
               cross_series_reducer: REDUCE_PERCENTILE_99
               group_by_fields:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   memory_utilization:
     display_name: "[P3] <%-clan%> - Cloud Run | Memory Utilization"
     conditions:
@@ -91,12 +95,13 @@ cloud_run:
           duration: 300s
           aggregations:
             - alignment_period: 60s
-              per_series_aligner: ALIGN_MEAN
+              per_series_aligner: ALIGN_PERCENTILE_99
               cross_series_reducer: REDUCE_PERCENTILE_99
               group_by_fields:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
   startup_latency:
     display_name: "[P4] <%-clan%> - Cloud Run | Startup Latency"
     conditions:
@@ -116,6 +121,7 @@ cloud_run:
                 - resource.label.service_name
     documentation:
       subject: 'Service: `$${resource.label.service_name}`'
+      content: 'Review the service in Cloud Run: https://console.cloud.google.com/run?project=<%-projectId%>'
 cloud_scheduler:
   failed_job:
     display_name: "[P4] <%-clan%> - Cloud Scheduler | Job Failed"
@@ -135,6 +141,7 @@ cloud_scheduler:
                 - resource.label.job_id
     documentation:
       subject: 'Job: `$${resource.label.job_id}`'
+      content: 'Review the job in Cloud Scheduler: https://console.cloud.google.com/cloudscheduler?project=<%-projectId%>'
   failed_executions:
     display_name: "[P3] <%-clan%> - Cloud Scheduler | Failed Executions"
     conditions:
@@ -155,6 +162,7 @@ cloud_scheduler:
                 - resource.label.job_id
     documentation:
       subject: 'Job: `$${resource.label.job_id}`'
+      content: 'Review the job in Cloud Scheduler: https://console.cloud.google.com/cloudscheduler?project=<%-projectId%>'
 cloud_sql:
   cpu_over_65:
     display_name: "[P3] <%-clan%> - CloudSQL | CPU over 65%"
@@ -175,6 +183,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   cpu_over_85:
     display_name: "[P3] <%-clan%> - CloudSQL | CPU over 85%"
     conditions:
@@ -194,6 +203,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   cpu_over_90:
     display_name: "[P3] <%-clan%> - CloudSQL | CPU over 90%"
     conditions:
@@ -213,6 +223,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   memory_over_90:
     display_name: "[P2] <%-clan%> - CloudSQL | Memory utilization above 90%"
     conditions:
@@ -232,6 +243,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   query_over_1s:
     display_name: "[P4] <%-clan%> - CloudSQL | Slow query"
     conditions:
@@ -251,6 +263,7 @@ cloud_sql:
                 - resource.label.resource_id
     documentation:
       subject: 'Database: `$${resource.label.resource_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   postgresql_connections:
     display_name: "[P3] <%-clan%> - CloudSQL | PostgreSQL Connections"
     conditions:
@@ -269,6 +282,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
   aggregated_lock_time:
     display_name: "[P4] <%-clan%> - CloudSQL | Aggregated Lock Time"
     conditions:
@@ -287,6 +301,7 @@ cloud_sql:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Cloud SQL: https://console.cloud.google.com/sql/instances?project=<%-projectId%>'
 firestore:
   request_latencies:
     display_name: "[P3] <%-clan%> - Firestore | High request latencies"
@@ -306,6 +321,7 @@ firestore:
                 - resource.label.database_id
     documentation:
       subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Firestore: https://console.cloud.google.com/firestore/databases?project=<%-projectId%>'
   failed_commits:
     display_name: "[P3] <%-clan%> - Firestore | High failed commits errors"
     conditions:
@@ -324,6 +340,9 @@ firestore:
               cross_series_reducer: REDUCE_SUM
               group_by_fields:
                 - metric.label."response_code"
+    documentation:
+      subject: 'Database: `$${resource.label.database_id}`'
+      content: 'Review the database in Firestore: https://console.cloud.google.com/firestore/databases?project=<%-projectId%>'
 memorystore:
   memory_over_50:
     display_name: "[P3] <%-clan%> - Memorystore | Memory over 50%"
@@ -344,6 +363,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   memory_over_75:
     display_name: "[P3] <%-clan%> - Memorystore | Memory over 75%"
     conditions:
@@ -363,6 +383,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   memory_over_90:
     display_name: "[P2] <%-clan%> - Memorystore | Memory over 90%"
     conditions:
@@ -382,6 +403,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   cpu_utilization:
     display_name: "[P2] <%-clan%> - Memorystore | CPU Utilization"
     conditions:
@@ -401,6 +423,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   system_memory_overload_duration:
     display_name: "[P1] <%-clan%> - Memorystore | System Memory Overload Duration"
     conditions:
@@ -419,6 +442,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
   calls:
     display_name: "[P3] <%-clan%> - Memorystore | Calls"
     conditions:
@@ -438,6 +462,7 @@ memorystore:
                 - resource.label.instance_id
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`'
+      content: 'Review the instance in Memorystore: https://console.cloud.google.com/memorystore/redis/instances?project=<%-projectId%>'
 pub_sub:
   unacknowledged_messages:
     display_name: "[P3] <%-clan%> - Pub/Sub | Undelivered message(s)"
@@ -457,6 +482,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   messages_in_dlq:
     display_name: "[P3] <%-clan%> - Pub/Sub | Message(s) in DLQ"
     conditions:
@@ -477,6 +503,7 @@ pub_sub:
                 - metric.label.response_code
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`, Response code: `$${metric.label.response_code}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   latency:
     display_name: "[P3] <%-clan%> - Pub/Sub | Response latency distribution"
     conditions:
@@ -496,6 +523,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   oldest_unacked_message_age:
     display_name: "[P3] <%-clan%> - Pub/Sub | Oldest unacked message age"
     conditions:
@@ -514,6 +542,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   push_requests_ack:
     display_name: "[P3] <%-clan%> - Pub/Sub | Push requests ack"
     conditions:
@@ -533,6 +562,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   push_requests_success:
     display_name: "[P3] <%-clan%> - Pub/Sub | Push requests success"
     conditions:
@@ -552,6 +582,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
   delivery_latency_health_score:
     display_name: "[P3] <%-clan%> - Pub/Sub | Delivery latency health score"
     conditions:
@@ -571,6 +602,7 @@ pub_sub:
                 - resource.label.subscription_id
     documentation:
       subject: 'Subscription: `$${resource.label.subscription_id}`'
+      content: 'Review the subscription in Pub/Sub: https://console.cloud.google.com/cloudpubsub/subscriptions?project=<%-projectId%>'
 spanner:
   cpu_utilization_by_priority:
     display_name: "[P2] <%-clan%> - Spanner | CPU Utilization"
@@ -591,6 +623,7 @@ spanner:
                 - metric.label.database
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`, Database: `$${resource.label.database}`'
+      content: 'Review the instance in Spanner: https://console.cloud.google.com/spanner/instances?project=<%-projectId%>'
   api_request_error_rate:
     display_name: "[P2] <%-clan%> - Spanner | API request error rate"
     conditions:
@@ -611,6 +644,7 @@ spanner:
                 - resource.label.database
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`, Database: `$${resource.label.database}`'
+      content: 'Review the instance in Spanner: https://console.cloud.google.com/spanner/instances?project=<%-projectId%>'
   request_latencies:
     display_name: '[P3] <%-clan%> - Spanner | API transaction latency'
     conditions:
@@ -631,3 +665,4 @@ spanner:
               per_series_aligner: ALIGN_PERCENTILE_95
     documentation:
       subject: 'Instance: `$${resource.label.instance_id}`, Database: `$${resource.label.database}}`'
+      content: 'Review the instance in Spanner: https://console.cloud.google.com/spanner/instances?project=<%-projectId%>'

package/dist/generators/common-resources/monitoring/validate.js

@@ -0,0 +1,58 @@
+const validate = {};
+
+const hasWhitespace = (str) => /\s/.test(str);
+
+const isUrl = (str) => {
+  try {
+    return Boolean(new URL(str));
+  } catch {
+    return false;
+  }
+};
+
+validate.clan = (input) => {
+  if (!hasWhitespace(input)) return true;
+  return 'Clan name must not include spaces';
+};
+
+validate.projectId = (input) => {
+  if (input.trim() !== '' && !hasWhitespace(input)) return true;
+  return 'Project ID must not be empty or contain spaces';
+};
+
+validate.hostname = (input) => {
+  if (input.split('.').at(-1) === 'com' && !hasWhitespace(input)) return true;
+  return 'Hostname must not include path to the page to run the check against or spaces';
+};
+
+validate.systemName = (input) => {
+  if (input.length === 3 && !hasWhitespace(input)) return true;
+  return 'System name must be 3 characters';
+};
+
+validate.confluenceUrl = (input) => {
+  if (input === '' || (isUrl(input) && !hasWhitespace(input))) return true;
+  return 'You must enter a valid URL';
+};
+
+validate.instanceID = (input) => {
+  if (input.split('/').length === 6 && !hasWhitespace(input)) return true;
+  return 'You must enter the full instance path (example: projects/example/locations/europe-west1/instances/instanceID)';
+};
+
+validate.databaseId = (input) => {
+  if (input.split(':').length === 2 && !hasWhitespace(input)) return true;
+  return 'You must enter the full database path (example: my-project:databaseID)';
+};
+
+validate.pubSubSubscription = (input) => {
+  if (input.split('/').length === 4 && !hasWhitespace(input)) return true;
+  return 'You must enter the full subscription path (example: projects/example/subscriptions/subscriptionId)';
+};
+
+validate.path = (input) => {
+  if (input.includes('/') && !hasWhitespace(input)) return true;
+  return 'Path must contain a slash and not include spaces';
+};
+
+module.exports = validate;

package/dist/generators/common-resources/pubsub/append.js

@@ -0,0 +1,130 @@
+const yaml = require('js-yaml');
+const fs = require('fs');
+
+const addDLQ = async (yamlArray, env, dlqTopic) => {
+  if (env === 'prod') {
+    yamlArray[0].dead_letter_topic = dlqTopic;
+  }
+};
+
+const appendNotIncludePull = async (inputs, subscriptionFilePath, dlqTopic) => {
+  const pullArray = [];
+
+  pullArray.push({
+    name: `${inputs.existingTopic}+${inputs.subscriberName}`,
+    ack_deadline_seconds: '60',
+    expiration_policy: '',
+  });
+
+  await addDLQ(pullArray, inputs.env, dlqTopic);
+  const yamlPullArray = yaml.dump(pullArray);
+  fs.appendFileSync(
+    subscriptionFilePath,
+    `pull_subscriptions:\n${yamlPullArray}`,
+  );
+};
+
+const appendIncludePull = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic,
+) => {
+  if (fileContent.includes('pull_subscriptions')) {
+    const pullArray = Object.values(originalContentYaml.pull_subscriptions);
+    const yamlPullArray = yaml.dump(pullArray);
+    fs.writeFileSync(
+      subscriptionFilePath,
+      `pull_subscriptions:\n${yamlPullArray}`,
+    );
+  }
+
+  const newPullArray = [];
+
+  newPullArray.push({
+    name: `${input.existingTopic}+${input.subscriberName}`,
+    ack_deadline_seconds: '60',
+    expiration_policy: '',
+  });
+
+  await addDLQ(newPullArray, input.env, dlqTopic);
+  const finalYamlPullArray = yaml.dump(newPullArray);
+  fs.appendFileSync(subscriptionFilePath, finalYamlPullArray);
+
+  if (fileContent.includes('push_subscriptions')) {
+    const pushArray = Object.values(originalContentYaml.push_subscriptions);
+    const yamlPushArray = yaml.dump(pushArray);
+    fs.appendFileSync(
+      subscriptionFilePath,
+      `push_subscriptions:\n${yamlPushArray}`,
+    );
+  }
+};
+
+const appendNotIncludePush = async (inputs, subscriptionFilePath, dlqTopic) => {
+  const pushArray = [];
+
+  pushArray.push({
+    name: `${inputs.existingTopic}+${inputs.subscriberName}`,
+    push_endpoint: inputs.pushEndpoint,
+    oidc_service_account_email: inputs.oidcEmail,
+    audience: inputs.audience,
+    expiration_policy: '',
+    filter: `attributes.Consumer = "${inputs.subscriberName}" OR NOT attributes:Consumer`,
+  });
+
+  await addDLQ(pushArray, inputs.env, dlqTopic);
+  const yamlPushArray = yaml.dump(pushArray);
+  fs.appendFileSync(
+    subscriptionFilePath,
+    `push_subscriptions:\n${yamlPushArray}`,
+  );
+};
+
+const appendIncludePush = async (
+  fileContent,
+  originalContentYaml,
+  subscriptionFilePath,
+  input,
+  dlqTopic,
+) => {
+  if (fileContent.includes('push_subscriptions')) {
+    const pushArray = Object.values(originalContentYaml.push_subscriptions);
+    const yamlPushArray = yaml.dump(pushArray);
+    fs.writeFileSync(
+      subscriptionFilePath,
+      `push_subscriptions:\n${yamlPushArray}`,
+    );
+  }
+
+  const newPushArray = [];
+
+  newPushArray.push({
+    name: `${input.existingTopic}+${input.subscriberName}`,
+    push_endpoint: input.pushEndpoint,
+    oidc_service_account_email: input.oidcEmail,
+    audience: input.audience,
+    expiration_policy: '',
+    filter: `attributes.Consumer = "${input.subscriberName}" OR NOT attributes:Consumer`,
+  });
+  await addDLQ(newPushArray, input.env, dlqTopic);
+  const yamlPushArray = yaml.dump(newPushArray);
+  fs.appendFileSync(subscriptionFilePath, yamlPushArray);
+
+  if (fileContent.includes('pull_subscriptions')) {
+    const pullArray = Object.values(originalContentYaml.pull_subscriptions);
+    const yamlPullArray = yaml.dump(pullArray);
+    fs.appendFileSync(
+      subscriptionFilePath,
+      `pull_subscriptions:\n${yamlPullArray}`,
+    );
+  }
+};
+
+module.exports = {
+  appendNotIncludePull,
+  appendIncludePull,
+  appendNotIncludePush,
+  appendIncludePush,
+};

package/dist/generators/common-resources/pubsub/get-gcp-projects.js

@@ -0,0 +1,34 @@
+const path = require('path');
+const fs = require('fs');
+
+const getKeyValue = (content, key) => {
+  let value = '';
+  content.split(/\r?\n/).forEach((line) => {
+    if (line.startsWith(key)) {
+      value = line.split('=')[1].replace('"', '').replace('"', '').trim();
+    }
+  });
+  return value;
+};
+
+const getProjectId = (env) => {
+  let projectId = '';
+  const projectHCL = path.join('infra', env, 'project.hcl');
+  if (fs.existsSync(projectHCL)) {
+    const projectHCLContent = fs.readFileSync(projectHCL, 'utf8');
+    projectId = getKeyValue(projectHCLContent, '  project_id');
+  }
+
+  return projectId;
+};
+
+const getProdPushEndopint = (pushEndpoint) => {
+  let prodPushEndopint = '';
+  prodPushEndopint = pushEndpoint.replace('.dev', '.com');
+  return prodPushEndopint;
+};
+
+module.exports = {
+  getProjectId,
+  getProdPushEndopint,
+};

package/dist/generators/common-resources/pubsub/handle-subscribers.js

@@ -0,0 +1,68 @@
+const fs = require('fs');
+const yaml = require('js-yaml');
+const {
+  appendNotIncludePull,
+  appendIncludePull,
+  appendNotIncludePush,
+  appendIncludePush,
+} = require('./append');
+
+const handleSubscribers = async (
+  env,
+  answers,
+  oidcEmail,
+  pushEndpoint,
+  subscriptionFilePath,
+  dlqTopic,
+) => {
+  const { subscriberName, existingTopic, pushOrPull, audience } = answers;
+
+  const subscriptionFileContent = fs.readFileSync(subscriptionFilePath, 'utf8');
+
+  const inputs = {
+    ...this.answers,
+    env,
+    existingTopic,
+    subscriberName,
+    audience,
+    oidcEmail,
+    pushEndpoint,
+    dlqTopic,
+  };
+
+  if (pushOrPull === 'pull') {
+    if (
+      subscriptionFileContent.length === 0 ||
+      !subscriptionFileContent.includes('pull_subscriptions')
+    ) {
+      await appendNotIncludePull(inputs, subscriptionFilePath, dlqTopic);
+    } else {
+      const originalContentYaml = yaml.load(subscriptionFileContent);
+      const fileContent = subscriptionFileContent;
+      await appendIncludePull(
+        fileContent,
+        originalContentYaml,
+        subscriptionFilePath,
+        inputs,
+        dlqTopic,
+      );
+    }
+  } else if (
+    subscriptionFileContent.length === 0 ||
+    !subscriptionFileContent.includes('push_subscriptions')
+  ) {
+    await appendNotIncludePush(inputs, subscriptionFilePath, dlqTopic);
+  } else {
+    const originalContentYaml = yaml.load(subscriptionFileContent);
+    const fileContent = subscriptionFileContent;
+    await appendIncludePush(
+      fileContent,
+      originalContentYaml,
+      subscriptionFilePath,
+      inputs,
+      dlqTopic,
+    );
+  }
+};
+
+module.exports = handleSubscribers;