@hiiretail/gcp-infra-cli 0.68.0 → 0.70.0

package/generators/docs/runbook/generator.json

@@ -0,0 +1,4 @@
+{
+  "name": "Runbook documentation",
+  "description": "Create a Runbook template"
+}

package/generators/docs/runbook/index.js

@@ -0,0 +1,42 @@
+const path = require('path');
+const chalk = require('chalk');
+const BaseGenerator = require('../../../src/BaseGenerator');
+const { required } = require('../../../src/validators');
+
+module.exports = class extends BaseGenerator {
+  prompting() {
+    const prompts = [
+      {
+        type: 'input',
+        name: 'system',
+        message: 'Name of the software system',
+        validate: required,
+      },
+    ];
+
+    return this.prompt(prompts).then((props) => {
+      this.answers = props;
+    });
+  }
+
+  writing() {
+    const { system } = this.answers;
+    const filename = this.kebabCase(system.toLowerCase());
+
+    this.copyDir(
+      'docs',
+      this.destinationPath(path.join('docs', filename)),
+      {
+        ...this.answers,
+        c4System: system.replace(/\s/g, ''),
+      },
+    );
+  }
+
+  end() {
+    this.log(`
+${chalk.green('Your Runbook documentation template have now been created.')}
+Next, push this change in a feature branch and open a pull request.
+`);
+  }
+};

package/generators/docs/runbook/templates/docs/runbook.md

@@ -0,0 +1,50 @@
+# Runbook for <%-system%>
+
+## General
+
+Describe in short what the purpose of the solution is.
+
+## Architecture
+
+Include C4 diagrams or links to the Software Guidebook.
+
+## Business Continuity and Disaster Recovery Plan
+
+Link to the Business Continuity and Disaster Recovery Plan documentation.
+
+## Services
+
+A short description of what the purpose of each service is. Links to the log files of all the services that are included in the solution.
+
+## Dashboard
+
+Links to one or multiple dashboards.
+
+## Service Level Objectives
+
+What are the SLOs?
+
+## Alerts
+
+What are the alerts that has been setup, where is alert sent to and what are the steps to mitigate the issue?
+
+## Health Checks
+
+Links to the configured uptime checks that has been setup in GCP.
+
+## How do I..?
+
+Good to know things. Such as `How do I check the price for a specific item?`
+
+## Known Issues
+
+Are there any known issues? If yes, what is the workaround to solve them?
+
+## Contact & Escalation Matrix
+
+If the team is unable to resolve the issue, who is the first in line to contact?
+
+| #  | Name | Role | E-Mail | Phone number |
+| --- | --- | --- | --- | --- |
+| 1 | Mr X | My Role | mr.x@x.com | 1234567 |
+| 2 | Mr Y | My Role | mr.y@y.com | 7654321 |

package/generators/resources/bigquery/index.js

@@ -7,24 +7,103 @@ module.exports = class extends BaseGenerator {
   prompting() {
     const prompts = [
       {
+        type: 'list',
+        name: 'resourceType',
+        message: 'Select type of resource you want to create',
+        default: 'datasets',
+        choices: ['scheduledQueries', 'datasets'],
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'queryName',
+        message: 'Please provide the name for the transfer config',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'dataSourceId',
+        message: 'Please provide the data source id. Cannot be changed once the transfer config is created.',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'destinationDatasetId',
+        message: 'Please provide the BigQuery target dataset id. Leave empty to specify later',
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'schedule',
+        message: 'Please provide the data transfer schedule. Format here: https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'notificationPubsubTopic',
+        message: 'Please provide Pub/Sub topic full name where notifications will be sent after transfer finish. Leave empty to specify later',
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'dataRefreshWindow',
+        message: 'Please provide the number of days to look back to automatically refresh the data. Leave empty to specify later',
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'list',
+        name: 'enableFailureEmail',
+        message: 'Select if you want to enable notifications to be sent to the email address of the user who owns this transfer config if the run fails',
+        default: 'false',
+        choices: ['false', 'true'],
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'dataPathTemplate',
+        message: 'Please provide the Cloud Storage URI that contains your files to be transferred',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'destinationTableTemplate',
+        message: 'Please provide the name of your destination table',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'fileFormat',
+        message: 'Please provide the type of files you want to transfe. Supported values CSV, JSON, AVRO, PARQUET, or ORC',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'datasets',
         type: 'input',
         name: 'datasetId',
         message: 'Please provide the dataset ID (must be alphanumeric, plus underscores)',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'input',
         name: 'datasetName',
         message: 'Please provide the user-frienldy name for the dataset',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'input',
         name: 'description',
         message: 'Please provide the description for the dataset',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'list',
         name: 'createTable',
         message: 'Do you want to create a table?',
@@ -32,20 +111,21 @@ module.exports = class extends BaseGenerator {
         choices: ['yes', 'no'],
       },
       {
-        when: (response) => response.createTable === 'yes',
+        when: (response) => response.createTable === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'tableId',
         message: 'Please provide the ID for the table',
         validate: required,
       },
       {
-        when: (response) => response.createTable === 'yes',
+        when: (response) => response.createTable === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'expirationTime',
         message: 'Please provide the time when the table expires (in milliseconds since the epoch). If set to `null`, the table will persist indefinitely.',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'list',
         name: 'createRoutine',
         message: 'Do you want to create a routine?',
@@ -53,21 +133,21 @@ module.exports = class extends BaseGenerator {
         choices: ['yes', 'no'],
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'routineId',
         message: 'Please provide the ID of the routine. The ID must contain only letters, numbers, or underscores',
         validate: required,
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'routineDescription',
         message: 'Please provide the description of the routine',
         validate: required,
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'list',
         name: 'routineType',
         message: 'Please select a routine type',
@@ -75,7 +155,7 @@ module.exports = class extends BaseGenerator {
         choices: ['SCALAR_FUNCTION', 'PROCEDURE'],
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'list',
         name: 'language',
         message: 'Please select a language of the routine',
@@ -83,7 +163,7 @@ module.exports = class extends BaseGenerator {
         choices: ['SQL', 'JAVASCRIPT'],
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'definitionBody',
         message: 'Please provide the body of the routine, specify null if not needed',
@@ -110,10 +190,21 @@ module.exports = class extends BaseGenerator {
       language,
       definitionBody,
       routineDescription,
+      resourceType,
+      dataSourceId,
+      queryName,
+      destinationDatasetId,
+      schedule,
+      notificationPubsubTopic,
+      dataRefreshWindow,
+      enableFailureEmail,
+      dataPathTemplate,
+      destinationTableTemplate,
+      fileFormat,
     } = this.answers;
 
     ['prod', 'staging'].forEach((env) => {
-      if (createTable === 'no') {
+      if (createTable === 'no' && resourceType === 'datasets') {
         this.copyDir(
           'bigquery',
           path.join('infra', env, 'bigquery', datasetId),
@@ -132,7 +223,9 @@ module.exports = class extends BaseGenerator {
             routineDescription,
           },
         );
-      } else {
+      }
+
+      if (createTable === 'yes' && resourceType === 'datasets') {
         ['schema', 'bigquery'].forEach((folder) => {
           this.copyDir(
             folder,
@@ -156,6 +249,27 @@ module.exports = class extends BaseGenerator {
           );
         });
       }
+
+      if (resourceType === 'scheduledQueries') {
+        this.copyDir(
+          'scheduled-queries',
+          path.join('infra', env, 'scheduled-queries'),
+          {
+            ...this.answers,
+            env,
+            dataSourceId,
+            queryName,
+            destinationDatasetId,
+            schedule,
+            notificationPubsubTopic,
+            dataRefreshWindow,
+            enableFailureEmail,
+            dataPathTemplate,
+            destinationTableTemplate,
+            fileFormat,
+          },
+        );
+      }
     });
   }
 
@@ -163,7 +277,7 @@ module.exports = class extends BaseGenerator {
     this.log(`
 ${chalk.green(`Your BigQuery resources have now been created. To finalize your configuration, please continue
 with manual editing of the generated files.`)}
-${chalk.green('1.')} Review and add modify schema.json file, change null values to needed ones.))}
+${chalk.green('1.')} Review and add modify schema.json file, change null values to needed ones.
 ${chalk.green('2.')} Push this change in a feature branch and open a pull request.
 `);
   }

package/generators/resources/bigquery/templates/bigquery/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-bigquery"
+  source = "git::https://github.com/terraform-google-modules/terraform-google-bigquery?ref=v5.4.0"
 }
 
 # Include all settings from the root terragrunt.hcl file

package/generators/resources/bigquery/templates/scheduled-queries/terragrunt.hcl

@@ -0,0 +1,56 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/extenda/terraform-google-bigquery//modules/scheduled_queries"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge(
+  local.project_vars.locals,
+  {
+    project_id = local.project_vars.locals.project_id
+    queries = [
+      {
+        name                   = "<%-queryName%>"
+        location               =  "EU"
+        data_source_id         = "<%-dataSourceId%>" # Cannot be changed once the transfer config is created.
+        <% if (destinationDatasetId == '') { %> destination_dataset_id = null <% } else { %>
+        destination_dataset_id = "<%-destinationDatasetId%>" <% } %>
+
+        # If the data source does not support a custom schedule, this should be empty.
+        # If it is empty, the default value for the data source will be used. The specified times are in UTC.
+        schedule = "<%-schedule%>"
+
+        # Pub/Sub topic where notifications will be sent after transfer runs associated with this transfer config finish.
+        <% if (notificationPubsubTopic == '') { %>
+        notification_pubsub_topic = null <% } else { %> notification_pubsub_topic = "<%-notificationPubsubTopic%>" <% } %>
+
+        # The number of days to look back to automatically refresh the data. Only valid if the data source supports the feature.
+        <% if (dataRefreshWindow == '') { %> data_refresh_window_days = null <% } else { %>
+        data_refresh_window_days = "<%-dataRefreshWindow%>" <% } %>
+
+        # When set to true, no runs are scheduled for a given transfer.
+        disabled = false
+
+        # Required parameters for Cloud Storage
+        params = {
+          data_path_template              = "<%-dataPathTemplate%>"
+          destination_table_name_template = "<%-destinationTableTemplate%>"
+          file_format                     = "<%-fileFormat%>"
+        }
+
+        # Email notifications will be sent according to these preferences to the email address of the user who owns this transfer config
+        enable_failure_email = <%-enableFailureEmail%>
+      }
+    ]
+  }
+)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.68.0",
+  "version": "0.70.0",
   "description": "Infrastructure as code generator for GCP.",
   "main": "src/cli.js",
   "bin": {