rake-terraform 0.0.8 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2I1MmU3MTY0MTcyYTZkNjljMzYxZWJkY2M3YzVlOGMwODA0YTU5YQ==
+    NWE5MWZmZDg3OGZlNWE4YmNkNmVhY2U3NTFkZGFmODhiMGMzOWE1MA==
   data.tar.gz: !binary |-
-    NzZlYjVmM2U4MjQ3OTg2Njk1NWJkMzA3MzlhOGU3OGJjMWM1OTQ0Zg==
+    NjA1MWMyZjY1YjljN2U0NjM3MTY3OTM2Y2I0YzlmNTQ4NzhhOTAyNA==
 SHA512:
   metadata.gz: !binary |-
-    MzcwMzQyOGVjMDM0ZjdhNWQ2OThjNzgwMDQ3MzVhYjY5NDQwZWQ0Mjg4ZGI3
-    OTljNjgyM2ZmMTFiOWQxNjZjOWEwNDM3M2UwMzIzNDNmZmNmNmNjNzljMDk0
-    NjRiZjRjMjJkYWYzNTVjOGMzYzk1NGQ4ODBiY2I4ODkwYjdkNjI=
+    YTFkNjY4MjY4MDkxNjNmZWNhYTQ1YTVmYWY4NWFhODkxYWY2YjJjZWE0YjM0
+    ZjEwMjE3M2YzMjc1NDU4ZjU0N2Y5YjQ3NDAzYTc2ODUwOWU0OGI4NGE2ZjE2
+    YmQzMWEwZWJkOWQwMmIxMTQyYmE4NmM5ODY0MDM3MWNiOWM2ZjE=
   data.tar.gz: !binary |-
-    YTFkMDkxZTc1ZWU4ZDNjOTk1NTg4Mjc1ZjE0OGFjNTdjNDBhMDg3ZTliNmQ5
-    OTc2NWFjM2Q5MDJmYzMxMzcyMDRhMjdlNGM2Y2ZkZjk0MGE0YjE4OTI4Y2E2
-    OTVjZTBmMDYwYjgzODNmZjk5YjhiYzk0ZGUxZThjYzBiMWFjMjU=
+    NDJmZDc5YjE2YTQwMTVmMTU0NjJiYTQ4Y2ZjZGUxOWNlMWY2YTZjMWEwNDY5
+    YmYwZmU2ZjEzNjNmMmU5YzQyNmJiMzIwMzllYmY0N2M0MTE1MWI0MDY0NmZm
+    N2NkYWMxZDE5ZmQ1YzZmYmFlMTU0ZjYzNWJiOGQ1YWY5MjFmZjc=

data/.gitignore

@@ -15,3 +15,22 @@ mkmf.log
 terraform/
 .idea/
 output/
+
+# ignore bundlers vendor dir
+/vendor/bundle/
+
+# Default ignores for vim editor
+.*.sw[a-z]
+*.un~
+Session.vim
+.netrwhist
+# Default ignores for emacs editor
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+.elc
+auto-save-list
+tramp
+.\#*
+

data/.rspec

@@ -0,0 +1,5 @@
+--color
+--tty
+--warnings
+--require spec_helper
+--format documentation

data/.rubocop.yml

@@ -0,0 +1,25 @@
+AllCops:
+  Exclude:
+    - 'spec/spec_helper.rb'
+    - 'vendor/**/*'
+Style/FileName:
+  Exclude:
+    # usability - module name should match the gem name
+    - 'lib/rake-terraform.rb'
+Metrics/AbcSize:
+  Exclude:
+    # TODO - fix this
+    - 'lib/rake-terraform/plan_task/task.rb'
+Metrics/MethodLength:
+  Max: 12
+Metrics/ModuleLength:
+  Exclude:
+    - 'spec/unit/*_spec.rb'
+Metrics/CyclomaticComplexity:
+  Exclude:
+      # TODO: fix tf_plan method to support splat arguments and break
+      # down into one or more command constructor methods
+      - 'lib/rake-terraform/terraformcmd.rb'
+Metrics/PerceivedComplexity:
+  Exclude:
+      - 'lib/rake-terraform/terraformcmd.rb'

data/.travis.yml

@@ -1,5 +1,6 @@
 language: ruby
 rvm:
+- 2.2.3
 - 2.1.0
 - 2.0.0
 deploy:

data/README.md

@@ -52,7 +52,7 @@ You can set the following configuration for the task:
 * `t.execution_path`: The path from which to execute the plan (default: `./terraform`)
     * This is useful if you are referencing cloud-config files using relative paths
 
-To use these tasks, you should `require 'rake_terraform'` at the top of your Rakefile.
+To use these tasks, you should `require 'rake-terraform'` at the top of your Rakefile.
 
 ### Default Tasks
 
@@ -96,6 +96,98 @@ The following environment variables can be set to tweak `default_task`'s behavio
 * `ENV['TERRAFORM_ENVIRONMENT_GLOB']` - Dir glob used to discover terraform environments (default: `terraform/**/*.tf`)
 * `ENV['TERRAFORM_OUTPUT_BASE']` - Directory to which plan files are saved/read. The environment name is appended to this automatically (default: `output/terraform`)
 * `ENV['TERRAFORM_CREDENTIAL_FILE']` - The path to your AWS credentials file (default: `~/.aws/credentials`)
+* `ENV['TERRAFORM_UNIQUE_STATE']` - Whether to use a unique state for this run. Requires `TERRAFORM_STATE_FILE` OR `TERRAFORM_STATE_DIR_VAR`. Can be any truthy or falsey looking string from [this list][wannabe_bool_string] (e.g `TRUE` or `FALSE`) 
+* `ENV['TERRAFORM_STATE_FILE']` - The full path to a state file to use for this run. Only used when `TERRAFORM_UNIQUE_STATE` is true, and cannot be used in conjunction with `TERRAFORM_STATE_DIR_VAR`. 
+* `ENV['TERRAFORM_STATE_DIR_VAR']` - The name of an environment variable that holds a variable that will be used to reference a directory in which to store state files in for this run. This directory will be a subdirectory within the terraform environment. Only used when `TERRAFORM_STATE_DIR` is true, and cannot be used in conjunction with `TERRAFORM_STATE_FILE`
+
+[wannabe_bool_string]: https://github.com/prodis/wannabe_bool#string
+
+#### Unique States
+
+Bu default, `rake-terraform` stores state within a given environment directory.
+
+Sometimes, you will have several infrastructure environments ("infrastructure
+environment" in this block here taken to mean e.g "staging" or "production"
+rather than the broader "terraform environment" used more generally in this
+doc) that are relatively homogeneous in terms of resources, where all changes
+are rolled out through those infrastructure environments in a cascading manner.
+Through application of [variable interpolation][tf_doc_var_interpol] and other
+methods, you can provide differing configuration for each of your resources to
+match those infrastructure environments.  The issue with this is that Terraform
+does not support resource names as variables, so when you come to apply the
+same resource layout with differing configuration to the next infrastructure
+environment, Terraform will see those configuration changes as needing to be
+applied to the existing deployed resources.
+
+One solution to this problem is to keep each of your infrastructure
+environments in separate directories ("terraform environments"), each with
+their own state file. An issue with this solution is that it does not confirm
+to DRY principles, and also depends on you manually diffing changes between
+files etc, rather than relying on `git diff` or similar.  Another solution
+might be to use separate divergent git branches, and cherry-pick relevant
+commits between them. Again, depends on clean commit hygiene and easy to mess
+up manual steps.
+
+By using a unique state file for each of your infrastructure environments,
+whilst utilising a single terraform environment, you can avoid repeating
+yourself and manage roll out changes to each of your infrastructure
+environments better.
+
+To enable unique state files, you need to set the environment variable
+`TERRAFORM_UNIQUE_STATE` to a [truthy value][wannabe_bool_string], then you
+need to EITHER set `TERRAFORM_STATE_FILE` to the full path of your chosen state
+file, OR set `TERRAFORM_STATE_DIR_VAR` to the name of another environment
+variable containing the name of your infrastructure environment.
+
+[tf_doc_var_interpol]: https://www.terraform.io/docs/configuration/interpolation.html
+
+##### Examples
+
+Use a specific state file
+
+    $ export TERRAFORM_UNIQUE_STATE="TRUE"
+    $ export TERRAFORM_STATE_FILE="/home/dave/.tf_states/staging/web_tier.tfstate"
+    $ bundle exec rake terraform:plan_web_tier
+    ...
+
+Use a variable to lookup the state file directory
+
+    $ export TF_VAR_infra_env="staging"
+    $ export TERRAFORM_UNIQUE_STATE="TRUE"
+    $ export TERRAFORM_STATE_DIR_VAR="TF_VAR_infra_env"
+    $ bundle exec rake terraform:plan_web_tier
+    ...
+
+This would result in a directory layout resembling the following:
+
+    terraform
+      web_tier
+        main.tf
+        variables.tf
+        output.tf
+        state
+          staging
+            terraform.tfstate
+            terraform.tfstate.backup
+          production 
+            terraform.tfstate
+            terraform.tfstate.backup
+      app_tier
+        main.tf
+        variables.tf
+        output.tf
+        state
+          staging
+            terraform.tfstate
+            terraform.tfstate.backup
+          production 
+            terraform.tfstate
+            terraform.tfstate.backup
+
+## Testing
+
+There is currently a basic rspec-based test harness in place. The default task
+runs unit tests and rubocop tests.
 
 ## Contributing
 

data/Rakefile

@@ -1,7 +1,26 @@
 require 'bundler/gem_tasks'
-require 'rake-terraform/default_tasks'
 require 'rubocop/rake_task'
+require 'rspec/core/rake_task'
+require 'rake-terraform'
+# include project rake tasks
+require 'rake-terraform/default_tasks'
+
+namespace :test do
+  # default unit tests
+  desc 'Run all unit tests in default spec suite'
+  RSpec::Core::RakeTask.new(:unit) do |task|
+    task.pattern = 'spec/(unit|rake-terraform)/*_spec.rb'
+  end
+  # add rubocop tasks to test namespace
+  RuboCop::RakeTask.new
+  # run all non-integration/default tests
+  desc 'Run all default test suites'
+  task default: [:unit, :rubocop]
+end
 
-task default: :rubocop
+# default task is to run all the default test suites
+task default: ['test:default']
 
-RuboCop::RakeTask.new
+# keep a rubocop alias in place for backwards compatibility
+desc 'Alias for test:rubocop'
+task rubocop: ['test:rubocop']

data/lib/{rake_terraform.rb → rake-terraform.rb}

@@ -1,5 +1,6 @@
 require 'rake-terraform/version'
 require 'rake-terraform/dsl'
+require 'rake-terraform/errors'
 
 # Top-level module
 module RakeTerraform

data/lib/rake-terraform/apply_task/config.rb

@@ -0,0 +1,47 @@
+require 'rake-terraform/env_process'
+
+module RakeTerraform
+  module ApplyTask
+    # Configuration data for terraform apply task
+    class Config
+      prepend RakeTerraform::EnvProcess
+
+      attr_writer :plan
+
+      def initialize
+        # initialize RakeTerraform::EnvProcess
+        super
+      end
+
+      def execution_path
+        @execution_path ||= File.expand_path 'terraform'
+      end
+
+      # setter method for execution_path triggers setters for tf_environment and
+      # state_file so that these are dynamically updated on change (but only if
+      # we are using directory state, and not explicit path to a state file)
+      def execution_path=(dir)
+        @tf_environment = dir
+        @state_file = tf_state_file if @state_dir
+        @execution_path = dir
+      end
+
+      def plan
+        @plan ||= File.expand_path(default_plan)
+      end
+
+      def opts
+        Map.new(plan: plan,
+                execution_path: execution_path,
+                unique_state: unique_state,
+                state_file: state_file)
+      end
+
+      private
+
+      def default_plan
+        File.join('output', 'terraform', 'plan.tf')
+      end
+    end
+  end
+end

data/lib/rake-terraform/{tasks/applytask.rb → apply_task/task.rb}

@@ -1,53 +1,41 @@
+require 'highline/import'
+require 'rake-terraform/basetask'
+require 'rake-terraform/terraformcmd'
+
 module RakeTerraform
   module ApplyTask
-    # Configuration data for terraform apply task
-    class Config
-      attr_accessor :plan
-      attr_accessor :execution_path
-      def initialize
-        @plan = File.expand_path(default_plan)
-        @execution_path = File.expand_path 'terraform'
-      end
-
-      def opts
-        Map.new(plan: @plan,
-                execution_path: @execution_path)
-      end
-
-      private
-
-      def default_plan
-        File.join('output', 'terraform', 'plan.tf')
-      end
-    end
-
     # Custom rake task to run `terraform apply`
     class Task < BaseTask
+      include RakeTerraform::TerraformCmd
+
       def initialize(opts)
         @opts = opts
       end
 
       def execute
-        require 'highline/import'
-
         plan = @opts.get(:plan)
-        validate_terraform_installed
-        ensure_plan_exists plan
-
-        system "terraform show  --module-depth=2 #{plan}"
-
-        say 'The above changes will be applied to your environment.'
-        exit unless agree 'Are you sure you want to execute this plan? (y/n)'
-
+        pre_execute_checks(plan)
         Dir.chdir(@opts.get(:execution_path)) do
-          system "terraform apply #{plan}"
+          if @opts[:unique_state]
+            tf_apply(plan, @opts[:state_file])
+          else
+            tf_apply(plan)
+          end
         end
       end
 
       private
 
+      # run pre execution checks
+      def pre_execute_checks(plan = @opts.get(:plan))
+        validate_terraform_installed
+        ensure_plan_exists plan
+        tf_show(plan)
+        say 'The above changes will be applied to your environment.'
+        exit unless agree 'Are you sure you want to execute this plan? (y/n)'
+      end
+
       def ensure_plan_exists(plan)
-        require 'fileutils'
         fail "Plan #{plan} does not exist! Aborting!" unless File.exist? plan
       end
     end

data/lib/rake-terraform/applytask.rb

@@ -0,0 +1,11 @@
+require 'rake-terraform/apply_task/config'
+require 'rake-terraform/apply_task/task'
+
+module RakeTerraform
+  # == RakeTerraform::ApplyTask
+  #
+  # Helper classes for applying a terraform plan
+  #
+  module ApplyTask
+  end
+end

data/lib/rake-terraform/{tasks/basetask.rb → basetask.rb}

@@ -1,4 +1,5 @@
 require 'rake/task'
+require 'fileutils'
 require 'iniparse'
 require 'English'
 
@@ -7,7 +8,7 @@ module RakeTerraform
   class BaseTask < Rake::Task
     def validate_terraform_installed
       error = 'Please ensure you have terraform installed and on your path!'
-      fail error unless terraform_installed?
+      fail TerraformNotInstalled, error unless terraform_installed?
     end
 
     def terraform_installed?

data/lib/rake-terraform/default_tasks.rb

@@ -1,7 +1,11 @@
-require 'rake_terraform'
+require 'rake-terraform'
 require 'pathname'
 
 namespace :terraform do
+  # TODO: refactor all environment variable processing into
+  #       RakeTerraform::EnvProcess, include in this task and pass to the
+  #       relevant Config classes, rather than including directly in the Config
+  #       classes
   env_glob = ENV['TERRAFORM_ENVIRONMENT_GLOB'] || 'terraform/**/*.tf'
   output_base = ENV['TERRAFORM_OUTPUT_BASE'] || 'output/terraform'
   credential_file = ENV['TERRAFORM_CREDENTIAL_FILE']
@@ -25,7 +29,6 @@ namespace :terraform do
 
     short_name = abs_relative_path.to_s.tr('/', '_')
     plan_path = File.expand_path File.join(output_base, "#{short_name}.tf")
-
     desc "Plan migration of #{short_name}" if hide_tasks == 'false'
     terraform_plan "plan_#{short_name}" do |t|
       t.input_dir = env

data/lib/rake-terraform/dsl.rb

@@ -7,7 +7,7 @@ module RakeTerraform
   # Definitions of methods for custom rake tasks
   module DSL
     def terraform_plan(*args)
-      require 'rake-terraform/tasks/plantask'
+      require 'rake-terraform/plantask'
       Rake::Task.define_task(*args) do
         c = RakeTerraform::PlanTask::Config.new
         yield c
@@ -16,7 +16,7 @@ module RakeTerraform
     end
 
     def terraform_apply(*args)
-      require 'rake-terraform/tasks/applytask'
+      require 'rake-terraform/applytask'
       Rake::Task.define_task(*args) do
         c = RakeTerraform::ApplyTask::Config.new
         yield c

data/lib/rake-terraform/env_process.rb

@@ -0,0 +1,142 @@
+require 'wannabe_bool'
+
+module RakeTerraform
+  # == RakeTerraform::EnvProcess
+  #
+  # Mixin for processing environment variables
+  #
+  # TODO: refactor all non accessor methods as private methods
+  #
+  module EnvProcess
+    attr_reader :unique_state, :state_file, :state_dir_var, :state_dir,
+                :tf_environment
+
+    def initialize
+      @unique_state, @state_file, @state_dir_var, @state_dir = nil
+      @tf_environment = nil
+      tf_unique_state_valid? && @unique_state = tf_unique_state
+      tf_state_dir_var_valid? && @state_dir_var = tf_state_dir_var
+      # tf_state_file represents the full path to the calculated file within
+      # tf_state_dir if given
+      if tf_state_dir_valid?
+        @state_dir = tf_state_dir
+        @state_file = tf_state_file
+      end
+      tf_state_file_valid? && @state_file = tf_state_file
+    end
+
+    # whether or not unique states are enabled and required args are also given
+    def tf_unique_state
+      state_var = ENV['TERRAFORM_UNIQUE_STATE'].to_b
+      return false if state_var == false
+      unless tf_unique_state_valid?
+        fail(
+          ArgumentError,
+          'Both or neither of TERRAFORM_STATE_FILE or TERRAFORM_STATE_DIR_VAR' \
+          ' given, or missing target for TERRAFORM_STATE_DIR_VAR'
+        )
+      end
+      ENV['TERRAFORM_UNIQUE_STATE'].to_b
+    end
+
+    # if we are using tf_state_dir_var and that is valid, then return the full
+    # path to the calculated state file. Otherwise return the value of a valid
+    # TERRAFORM_STATE_FILE variable
+    def tf_state_file
+      return state_dir_full_path if tf_state_dir_valid?
+      return nil if ENV['TERRAFORM_STATE_FILE'].nil?
+      unless tf_state_file_valid?
+        fail(
+          ArgumentError,
+          'Argument for TERRAFORM_STATE_FILE is invalid'
+        )
+      end
+      ENV['TERRAFORM_STATE_FILE']
+    end
+
+    # return the value if tf_state_dir_var
+    # see also: tf_state_dir
+    def tf_state_dir_var
+      return nil if ENV['TERRAFORM_STATE_DIR_VAR'].nil?
+      unless tf_state_dir_var_valid?
+        fail(
+          ArgumentError,
+          'Argument for TERRAFORM_STATE_DIR_VAR is invalid'
+        )
+      end
+      ENV['TERRAFORM_STATE_DIR_VAR']
+    end
+
+    # return the target of tf_state_dir_var
+    # see also: tf_state_dir_var
+    def tf_state_dir
+      return nil if ENV['TERRAFORM_STATE_DIR_VAR'].nil?
+      unless tf_state_dir_valid?
+        fail(
+          ArgumentError,
+          'Argument for TERRAFORM_STATE_DIR_VAR is invalid'
+        )
+      end
+      dir_var = ENV['TERRAFORM_STATE_DIR_VAR']
+      "#{tf_env_string}state/#{ENV[dir_var]}"
+    end
+
+    # calculate the full path to a state file within tf_state_dir
+    def state_dir_full_path(dir = tf_state_dir)
+      File.expand_path(
+        File.join(dir, default_state_file_name)
+      )
+    end
+
+    # if @tf_environment is set then return that, postfixed by a '/' -
+    # otherwise return 'terraform/'
+    def tf_env_string
+      if @tf_environment
+        "#{@tf_environment}/"
+      else
+        'terraform/'
+      end
+    end
+
+    # validate tf_unique_state
+    def tf_unique_state_valid?
+      state_var = ENV['TERRAFORM_UNIQUE_STATE'].to_b
+      return true if state_var == false
+      if tf_state_file_valid? && tf_state_dir_var_valid?
+        return false
+      else
+        tf_state_file_valid? || tf_state_dir_var_valid?
+      end
+    end
+
+    # validate tf_state_file_valid?
+    # returns false if no environment set, or the file does not have a single
+    # a-z0-9 char
+    # TODO: improve regex?
+    def tf_state_file_valid?
+      return false if ENV['TERRAFORM_STATE_FILE'].nil?
+      ENV['TERRAFORM_STATE_FILE'] =~ /[a-z0-9]/i
+    end
+
+    # validate tf_state_dir_var (and corresponding target) is valid
+    # returns false if
+    #  * TERRAFORM_STATE_DIR_VAR is nil
+    #  * We cannot find the variable referenced by TERRAFORM_STATE_DIR_VAR
+    #  * The variable referenced contains something other than a-z0-9_ chars
+    def tf_state_dir_var_valid?
+      return false if ENV['TERRAFORM_STATE_DIR_VAR'].nil?
+      dir_var = ENV['TERRAFORM_STATE_DIR_VAR']
+      return false unless dir_var =~ /^[a-z0-9_]+$/i
+      value = ENV[dir_var]
+      return false if value.nil?
+      value =~ /^[a-z0-9_]+$/i
+    end
+
+    alias_method :tf_state_dir_valid?, :tf_state_dir_var_valid?
+
+    # name of the default state file
+    def default_state_file_name
+      'terraform.tfstate'
+    end
+  end
+end