@hiiretail/gcp-infra-cli 0.67.2 → 0.69.0

package/generators/organization/clan-project/templates/clan-project/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v1.0.2"
+  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v1.0.6"
 }
 
 dependency "parent_folder" {

package/generators/resources/bigquery/index.js

@@ -7,24 +7,103 @@ module.exports = class extends BaseGenerator {
   prompting() {
     const prompts = [
       {
+        type: 'list',
+        name: 'resourceType',
+        message: 'Select type of resource you want to create',
+        default: 'datasets',
+        choices: ['scheduledQueries', 'datasets'],
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'queryName',
+        message: 'Please provide the name for the transfer config',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'dataSourceId',
+        message: 'Please provide the data source id. Cannot be changed once the transfer config is created.',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'destinationDatasetId',
+        message: 'Please provide the BigQuery target dataset id. Leave empty to specify later',
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'schedule',
+        message: 'Please provide the data transfer schedule. Format here: https://cloud.google.com/appengine/docs/flexible/python/scheduling-jobs-with-cron-yaml#the_schedule_format',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'notificationPubsubTopic',
+        message: 'Please provide Pub/Sub topic full name where notifications will be sent after transfer finish. Leave empty to specify later',
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'dataRefreshWindow',
+        message: 'Please provide the number of days to look back to automatically refresh the data. Leave empty to specify later',
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'list',
+        name: 'enableFailureEmail',
+        message: 'Select if you want to enable notifications to be sent to the email address of the user who owns this transfer config if the run fails',
+        default: 'false',
+        choices: ['false', 'true'],
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'dataPathTemplate',
+        message: 'Please provide the Cloud Storage URI that contains your files to be transferred',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'destinationTableTemplate',
+        message: 'Please provide the name of your destination table',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'scheduledQueries',
+        type: 'input',
+        name: 'fileFormat',
+        message: 'Please provide the type of files you want to transfe. Supported values CSV, JSON, AVRO, PARQUET, or ORC',
+        validate: required,
+      },
+      {
+        when: (response) => response.resourceType === 'datasets',
         type: 'input',
         name: 'datasetId',
         message: 'Please provide the dataset ID (must be alphanumeric, plus underscores)',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'input',
         name: 'datasetName',
         message: 'Please provide the user-frienldy name for the dataset',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'input',
         name: 'description',
         message: 'Please provide the description for the dataset',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'list',
         name: 'createTable',
         message: 'Do you want to create a table?',
@@ -32,20 +111,21 @@ module.exports = class extends BaseGenerator {
         choices: ['yes', 'no'],
       },
       {
-        when: (response) => response.createTable === 'yes',
+        when: (response) => response.createTable === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'tableId',
         message: 'Please provide the ID for the table',
         validate: required,
       },
       {
-        when: (response) => response.createTable === 'yes',
+        when: (response) => response.createTable === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'expirationTime',
         message: 'Please provide the time when the table expires (in milliseconds since the epoch). If set to `null`, the table will persist indefinitely.',
         validate: required,
       },
       {
+        when: (response) => response.resourceType === 'datasets',
         type: 'list',
         name: 'createRoutine',
         message: 'Do you want to create a routine?',
@@ -53,21 +133,21 @@ module.exports = class extends BaseGenerator {
         choices: ['yes', 'no'],
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'routineId',
         message: 'Please provide the ID of the routine. The ID must contain only letters, numbers, or underscores',
         validate: required,
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'routineDescription',
         message: 'Please provide the description of the routine',
         validate: required,
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'list',
         name: 'routineType',
         message: 'Please select a routine type',
@@ -75,7 +155,7 @@ module.exports = class extends BaseGenerator {
         choices: ['SCALAR_FUNCTION', 'PROCEDURE'],
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'list',
         name: 'language',
         message: 'Please select a language of the routine',
@@ -83,7 +163,7 @@ module.exports = class extends BaseGenerator {
         choices: ['SQL', 'JAVASCRIPT'],
       },
       {
-        when: (response) => response.createRoutine === 'yes',
+        when: (response) => response.createRoutine === 'yes' && response.resourceType === 'datasets',
         type: 'input',
         name: 'definitionBody',
         message: 'Please provide the body of the routine, specify null if not needed',
@@ -110,10 +190,21 @@ module.exports = class extends BaseGenerator {
       language,
       definitionBody,
       routineDescription,
+      resourceType,
+      dataSourceId,
+      queryName,
+      destinationDatasetId,
+      schedule,
+      notificationPubsubTopic,
+      dataRefreshWindow,
+      enableFailureEmail,
+      dataPathTemplate,
+      destinationTableTemplate,
+      fileFormat,
     } = this.answers;
 
     ['prod', 'staging'].forEach((env) => {
-      if (createTable === 'no') {
+      if (createTable === 'no' && resourceType === 'datasets') {
         this.copyDir(
           'bigquery',
           path.join('infra', env, 'bigquery', datasetId),
@@ -132,7 +223,9 @@ module.exports = class extends BaseGenerator {
             routineDescription,
           },
         );
-      } else {
+      }
+
+      if (createTable === 'yes' && resourceType === 'datasets') {
         ['schema', 'bigquery'].forEach((folder) => {
           this.copyDir(
             folder,
@@ -156,6 +249,27 @@ module.exports = class extends BaseGenerator {
           );
         });
       }
+
+      if (resourceType === 'scheduledQueries') {
+        this.copyDir(
+          'scheduled-queries',
+          path.join('infra', env, 'scheduled-queries'),
+          {
+            ...this.answers,
+            env,
+            dataSourceId,
+            queryName,
+            destinationDatasetId,
+            schedule,
+            notificationPubsubTopic,
+            dataRefreshWindow,
+            enableFailureEmail,
+            dataPathTemplate,
+            destinationTableTemplate,
+            fileFormat,
+          },
+        );
+      }
     });
   }
 
@@ -163,7 +277,7 @@ module.exports = class extends BaseGenerator {
     this.log(`
 ${chalk.green(`Your BigQuery resources have now been created. To finalize your configuration, please continue
 with manual editing of the generated files.`)}
-${chalk.green('1.')} Review and add modify schema.json file, change null values to needed ones.))}
+${chalk.green('1.')} Review and add modify schema.json file, change null values to needed ones.
 ${chalk.green('2.')} Push this change in a feature branch and open a pull request.
 `);
   }

package/generators/resources/bigquery/templates/bigquery/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-bigquery"
+  source = "git::https://github.com/terraform-google-modules/terraform-google-bigquery?ref=v5.4.0"
 }
 
 # Include all settings from the root terragrunt.hcl file

package/generators/resources/bigquery/templates/scheduled-queries/terragrunt.hcl

@@ -0,0 +1,56 @@
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder.
+terraform {
+  source = "git::https://github.com/terraform-google-modules/terraform-google-bigquery//modules/scheduled_queries?ref=v5.4.0"
+}
+
+# Include all settings from the root terragrunt.hcl file
+include {
+  path = find_in_parent_folders("terragrunt_root.hcl")
+}
+
+locals {
+  project_vars = read_terragrunt_config(find_in_parent_folders("project.hcl"))
+}
+
+# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
+inputs = merge(
+  local.project_vars.locals,
+  {
+    project_id = local.project_vars.locals.project_id
+    queries = [
+      {
+        name                   = "<%-queryName%>"
+        location               =  "EU"
+        data_source_id         = "<%-dataSourceId%>" # Cannot be changed once the transfer config is created.
+        <% if (destinationDatasetId == '') { %> destination_dataset_id = null <% } else { %>
+        destination_dataset_id = "<%-destinationDatasetId%>" <% } %>
+
+        # If the data source does not support a custom schedule, this should be empty.
+        # If it is empty, the default value for the data source will be used. The specified times are in UTC.
+        schedule = "<%-schedule%>"
+
+        # Pub/Sub topic where notifications will be sent after transfer runs associated with this transfer config finish.
+        <% if (notificationPubsubTopic == '') { %>
+        notification_pubsub_topic = null <% } else { %> notification_pubsub_topic = "<%-notificationPubsubTopic%>" <% } %>
+
+        # The number of days to look back to automatically refresh the data. Only valid if the data source supports the feature.
+        <% if (dataRefreshWindow == '') { %> data_refresh_window_days = null <% } else { %>
+        data_refresh_window_days = "<%-dataRefreshWindow%>" <% } %>
+
+        # When set to true, no runs are scheduled for a given transfer.
+        disabled = false
+
+        # Required parameters for Cloud Storage
+        params = {
+          data_path_template              = "<%-dataPathTemplate%>"
+          destination_table_name_template = "<%-destinationTableTemplate%>"
+          file_format                     = "<%-fileFormat%>"
+        }
+
+        # Email notifications will be sent according to these preferences to the email address of the user who owns this transfer config
+        enable_failure_email = <%-enableFailureEmail%>
+      }
+    ]
+  }
+)

package/generators/resources/kafka-connect/templates/elastic/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/extenda/tf-module-kafka//kafka-connect-ccloud?ref=v0.1.4"
+  source = "git::https://github.com/extenda/tf-module-kafka//kafka-connect-ccloud?ref=v1.0.0"
 }
 
 # Include all settings from the root terragrunt.hcl file

package/generators/resources/kafka-connect/templates/pubsub/terragrunt.hcl

@@ -1,7 +1,7 @@
 # Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
 # working directory, into a temporary folder, and execute your Terraform commands in that folder.
 terraform {
-  source = "git::https://github.com/extenda/tf-module-kafka//kafka-connect-ccloud?ref=v0.1.4"
+  source = "git::https://github.com/extenda/tf-module-kafka//kafka-connect-ccloud?ref=v1.0.0"
 }
 
 # Include all settings from the root terragrunt.hcl file

package/generators/tribe-resources/tribe-project/templates/project/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v0.9.8"
+  source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=v1.0.6"
 }
 
 dependency "tribe_folder" {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.67.2",
+  "version": "0.69.0",
   "description": "Infrastructure as code generator for GCP.",
   "main": "src/cli.js",
   "bin": {